@meith/import 0.16.0

package/src/phpbb-rows.ts

@@ -0,0 +1,160 @@
+export interface PhpbbUser {
+  readonly user_id: number
+  readonly user_type: number
+  readonly username: string
+  readonly user_email: string
+  readonly user_password: string
+  readonly user_regdate: number
+  readonly user_lastvisit: number
+  readonly user_posts: number
+  readonly group_id: number
+}
+
+export interface PhpbbForum {
+  readonly forum_id: number
+  readonly parent_id: number
+  readonly forum_name: string
+  readonly forum_desc: string
+  readonly forum_type: number
+  readonly forum_link: string
+  readonly left_id: number
+}
+
+export interface PhpbbTopic {
+  readonly topic_id: number
+  readonly forum_id: number
+  readonly topic_title: string
+  readonly topic_poster: number
+  readonly topic_first_poster_name: string
+  readonly topic_time: number
+  readonly topic_last_post_time: number
+  readonly topic_views: number
+  readonly topic_posts_approved: number
+  readonly topic_status: number
+  readonly topic_type: number
+  readonly topic_visibility: number
+  readonly topic_moved_id: number
+}
+
+export interface PhpbbPost {
+  readonly post_id: number
+  readonly topic_id: number
+  readonly forum_id: number
+  readonly poster_id: number
+  readonly post_username: string
+  readonly post_subject: string
+  readonly post_text: string
+  readonly bbcode_uid: string
+  readonly post_time: number
+  readonly post_edit_time: number
+  readonly post_visibility: number
+}
+
+export interface PhpbbAvatarRow {
+  readonly user_id: number
+  readonly user_avatar: string
+  readonly user_avatar_type: string
+  readonly user_avatar_width: number
+  readonly user_avatar_height: number
+}
+
+export interface PhpbbAttachment {
+  readonly attach_id: number
+  readonly post_msg_id: number
+  readonly poster_id: number
+  readonly real_filename: string
+  readonly physical_filename: string
+  readonly mimetype: string
+  readonly filesize: number
+  readonly download_count: number
+  readonly filetime: number
+  readonly thumbnail: number
+}
+
+export interface PhpbbPollTopic {
+  readonly topic_id: number
+  readonly poll_title: string
+  readonly poll_start: number
+  readonly poll_length: number
+  readonly poll_max_options: number
+  readonly poll_vote_change: number
+}
+
+export interface PhpbbPollOption {
+  readonly topic_id: number
+  readonly poll_option_id: number
+  readonly poll_option_text: string
+  readonly poll_option_total: number
+}
+
+export interface PhpbbPollVote {
+  readonly topic_id: number
+  readonly poll_option_id: number
+  readonly vote_user_id: number
+}
+
+export interface PhpbbPrivateMessage {
+  readonly msg_id: number
+  readonly author_id: number
+  readonly message_subject: string
+  readonly message_text: string
+  readonly bbcode_uid: string
+  readonly message_time: number
+}
+
+export interface PhpbbPrivateMessageCopy {
+  readonly msg_id: number
+  readonly user_id: number
+  readonly author_id: number
+  readonly folder_id: number
+  readonly pm_deleted: number
+  readonly pm_unread: number
+}
+
+export interface PhpbbWatch {
+  readonly user_id: number
+  readonly target_id: number
+}
+
+export interface PhpbbWarning {
+  readonly warning_id: number
+  readonly user_id: number
+  readonly post_id: number
+  readonly warning_time: number
+}
+
+export interface PhpbbBan {
+  readonly ban_id: number
+  readonly ban_userid: number
+  readonly ban_start: number
+  readonly ban_end: number
+  readonly ban_exclude: number
+  readonly ban_reason: string
+  readonly ban_give_reason: string
+}
+
+export interface PhpbbZebra {
+  readonly user_id: number
+  readonly zebra_id: number
+  readonly friend: number
+  readonly foe: number
+}
+
+export interface PhpbbTables {
+  readonly users?: readonly PhpbbUser[]
+  readonly forums?: readonly PhpbbForum[]
+  readonly topics?: readonly PhpbbTopic[]
+  readonly posts?: readonly PhpbbPost[]
+  readonly avatars?: readonly PhpbbAvatarRow[]
+  readonly attachments?: readonly PhpbbAttachment[]
+  readonly pollTopics?: readonly PhpbbPollTopic[]
+  readonly pollOptions?: readonly PhpbbPollOption[]
+  readonly pollVotes?: readonly PhpbbPollVote[]
+  readonly privateMessages?: readonly PhpbbPrivateMessage[]
+  readonly privateMessageCopies?: readonly PhpbbPrivateMessageCopy[]
+  readonly topicWatch?: readonly PhpbbWatch[]
+  readonly forumWatch?: readonly PhpbbWatch[]
+  readonly warnings?: readonly PhpbbWarning[]
+  readonly bans?: readonly PhpbbBan[]
+  readonly zebra?: readonly PhpbbZebra[]
+}

package/src/phpbb-text.ts

@@ -0,0 +1,44 @@
+const ENTITIES: Readonly<Record<string, string>> = {
+  '&amp;': '&',
+  '&lt;': '<',
+  '&gt;': '>',
+  '&quot;': '"',
+  '&#039;': "'",
+  '&#39;': "'",
+  '&nbsp;': ' ',
+}
+
+export function decodeHtmlEntities(text: string): string {
+  return text
+    .replace(/&#(\d+);/g, (whole, code: string) => {
+      const point = Number(code)
+      return Number.isSafeInteger(point) && point > 0 && point <= 0x10ffff
+        ? String.fromCodePoint(point)
+        : whole
+    })
+    .replace(/&(amp|lt|gt|quot|nbsp);/g, (whole) => ENTITIES[whole] ?? whole)
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+export function decodePhpbbText(text: string, bbcodeUid: string): string {
+  let out = text
+
+  if (bbcodeUid.trim() !== '') {
+    out = out.replace(new RegExp(`:${escapeRegExp(bbcodeUid.trim())}(?=[\\]:])`, 'g'), '')
+  }
+
+  out = out.replace(/<!-- s(\S+) -->.*?<!-- s\1 -->/gs, '$1')
+
+  out = out.replace(
+    /<!-- ([mwe]) --><a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a><!-- \1 -->/gs,
+    (_whole, kind: string, href: string, label: string) => {
+      const url = kind === 'e' ? href.replace(/^mailto:/, '') : href
+      return label === href || label === url || kind === 'e' ? url : `[url=${url}]${label}[/url]`
+    },
+  )
+
+  return decodeHtmlEntities(out)
+}

package/src/run.ts

@@ -0,0 +1,261 @@
+import type { ImportSource, Page } from './source'
+import type {
+  ImportedAttachment,
+  ImportedAvatar,
+  ImportedBan,
+  ImportedForum,
+  ImportedPoll,
+  ImportedPollVote,
+  ImportedPost,
+  ImportedPrivateMessage,
+  ImportedReputation,
+  ImportedSubscription,
+  ImportedThread,
+  ImportedUser,
+  ImportedUserRelation,
+  ImportedWarning,
+} from './types'
+
+export const KINDS = [
+  'users',
+  'forums',
+  'threads',
+  'posts',
+  'avatars',
+  'attachments',
+  'polls',
+  'pollVotes',
+  'privateMessages',
+  'threadSubscriptions',
+  'forumSubscriptions',
+  'reputation',
+  'warnings',
+  'bans',
+  'userRelations',
+] as const
+export type Kind = (typeof KINDS)[number]
+
+export type Cursors = Readonly<Record<Kind, number>>
+
+export const NO_PROGRESS: Cursors = Object.fromEntries(KINDS.map((kind) => [kind, 0])) as Record<
+  Kind,
+  number
+>
+
+export interface ImportSink {
+  putUsers(rows: readonly ImportedUser[]): Promise<WriteResult>
+  putForums(rows: readonly ImportedForum[]): Promise<WriteResult>
+  putThreads(rows: readonly ImportedThread[]): Promise<WriteResult>
+  putPosts(rows: readonly ImportedPost[]): Promise<WriteResult>
+  putAvatars(rows: readonly ImportedAvatar[]): Promise<WriteResult>
+  putAttachments(rows: readonly ImportedAttachment[]): Promise<WriteResult>
+  putPolls(rows: readonly ImportedPoll[]): Promise<WriteResult>
+  putPollVotes(rows: readonly ImportedPollVote[]): Promise<WriteResult>
+  putPrivateMessages(rows: readonly ImportedPrivateMessage[]): Promise<WriteResult>
+  putThreadSubscriptions(rows: readonly ImportedSubscription[]): Promise<WriteResult>
+  putForumSubscriptions(rows: readonly ImportedSubscription[]): Promise<WriteResult>
+  putReputation(rows: readonly ImportedReputation[]): Promise<WriteResult>
+  putWarnings(rows: readonly ImportedWarning[]): Promise<WriteResult>
+  putBans(rows: readonly ImportedBan[]): Promise<WriteResult>
+  putUserRelations(rows: readonly ImportedUserRelation[]): Promise<WriteResult>
+}
+
+export interface WriteResult {
+  readonly inserted: number
+  readonly updated: number
+  readonly skipped: readonly { readonly legacyId: number; readonly reason: string }[]
+}
+
+export interface KindReport {
+  readonly read: number
+  readonly inserted: number
+  readonly updated: number
+  readonly skipped: readonly { readonly legacyId: number; readonly reason: string }[]
+}
+
+export interface ImportReport {
+  readonly kinds: Readonly<Record<Kind, KindReport>>
+  readonly cursors: Cursors
+  readonly finished: boolean
+  readonly readThisRun: number
+}
+
+export interface RunOptions {
+  readonly source: ImportSource
+  readonly sink: ImportSink
+  readonly pageSize?: number | undefined
+  readonly budget?: number | undefined
+  readonly from?: Partial<Cursors> | undefined
+}
+
+const emptyKind = (): KindReport => ({ read: 0, inserted: 0, updated: 0, skipped: [] })
+
+export async function runImport(options: RunOptions): Promise<ImportReport> {
+  const pageSize = options.pageSize ?? 200
+  const budget = options.budget ?? 2000
+  const cursors: Record<Kind, number> = { ...NO_PROGRESS, ...options.from }
+
+  const kinds = Object.fromEntries(KINDS.map((kind) => [kind, emptyKind()])) as Record<
+    Kind,
+    KindReport
+  >
+
+  let readThisRun = 0
+  const exhausted = new Set<Kind>()
+
+  for (const kind of KINDS) {
+    while (readThisRun < budget) {
+      const limit = Math.min(pageSize, budget - readThisRun)
+      const outcome = await importPage(kind, options, cursors[kind], limit)
+
+      kinds[kind] = merge(kinds[kind], outcome.report)
+      readThisRun += outcome.report.read
+
+      if (outcome.nextCursor === null) {
+        exhausted.add(kind)
+        break
+      }
+      cursors[kind] = outcome.nextCursor
+    }
+  }
+
+  return {
+    kinds,
+    cursors,
+    finished: KINDS.every((kind) => exhausted.has(kind)),
+    readThisRun,
+  }
+}
+
+function merge(a: KindReport, b: KindReport): KindReport {
+  return {
+    read: a.read + b.read,
+    inserted: a.inserted + b.inserted,
+    updated: a.updated + b.updated,
+    skipped: [...a.skipped, ...b.skipped],
+  }
+}
+
+async function importPage(
+  kind: Kind,
+  options: RunOptions,
+  after: number,
+  limit: number,
+): Promise<{ report: KindReport; nextCursor: number | null }> {
+  const { source, sink } = options
+
+  const write = async <T>(
+    page: Page<T>,
+    put: (rows: readonly T[]) => Promise<WriteResult>,
+  ): Promise<{ report: KindReport; nextCursor: number | null }> => {
+    if (page.rows.length === 0) return { report: emptyKind(), nextCursor: page.nextCursor }
+    const written = await put(page.rows)
+    return { report: report(page.rows.length, written), nextCursor: page.nextCursor }
+  }
+
+  switch (kind) {
+    case 'users':
+      return write(await source.users(after, limit), sink.putUsers.bind(sink))
+    case 'forums':
+      return write(await source.forums(after, limit), sink.putForums.bind(sink))
+    case 'threads':
+      return write(await source.threads(after, limit), sink.putThreads.bind(sink))
+    case 'posts':
+      return write(await source.posts(after, limit), sink.putPosts.bind(sink))
+    case 'avatars':
+      return write(await source.avatars(after, limit), sink.putAvatars.bind(sink))
+    case 'attachments':
+      return write(await source.attachments(after, limit), sink.putAttachments.bind(sink))
+    case 'polls':
+      return write(await source.polls(after, limit), sink.putPolls.bind(sink))
+    case 'pollVotes':
+      return write(await source.pollVotes(after, limit), sink.putPollVotes.bind(sink))
+    case 'privateMessages':
+      return write(await source.privateMessages(after, limit), sink.putPrivateMessages.bind(sink))
+    case 'threadSubscriptions':
+      return write(
+        await source.threadSubscriptions(after, limit),
+        sink.putThreadSubscriptions.bind(sink),
+      )
+    case 'forumSubscriptions':
+      return write(
+        await source.forumSubscriptions(after, limit),
+        sink.putForumSubscriptions.bind(sink),
+      )
+    case 'reputation':
+      return write(await source.reputation(after, limit), sink.putReputation.bind(sink))
+    case 'warnings':
+      return write(await source.warnings(after, limit), sink.putWarnings.bind(sink))
+    case 'bans':
+      return write(await source.bans(after, limit), sink.putBans.bind(sink))
+    case 'userRelations':
+      return write(await source.userRelations(after, limit), sink.putUserRelations.bind(sink))
+  }
+}
+
+function report(read: number, written: WriteResult): KindReport {
+  return { read, inserted: written.inserted, updated: written.updated, skipped: written.skipped }
+}
+
+export interface CounterComparison {
+  readonly legacyId: number
+  readonly field: string
+  readonly claimed: number
+  readonly actual: number
+}
+
+export function compareCounters(input: {
+  readonly forums: readonly ImportedForum[]
+  readonly threads: readonly ImportedThread[]
+  readonly posts: readonly ImportedPost[]
+  readonly claimedForumTotals: Readonly<Record<number, { threads: number; posts: number }>>
+}): readonly CounterComparison[] {
+  const differences: CounterComparison[] = []
+
+  const visibleThreads = input.threads.filter((thread) => thread.visibility === 'visible')
+  const visiblePosts = input.posts.filter((post) => post.visibility === 'visible')
+
+  for (const forum of input.forums) {
+    const claimed = input.claimedForumTotals[forum.legacyId]
+    if (claimed === undefined) continue
+
+    const actualThreads = visibleThreads.filter(
+      (thread) => thread.legacyForumId === forum.legacyId,
+    ).length
+    const actualPosts = visiblePosts.filter((post) => post.legacyForumId === forum.legacyId).length
+
+    if (claimed.threads !== actualThreads) {
+      differences.push({
+        legacyId: forum.legacyId,
+        field: 'threads',
+        claimed: claimed.threads,
+        actual: actualThreads,
+      })
+    }
+    if (claimed.posts !== actualPosts) {
+      differences.push({
+        legacyId: forum.legacyId,
+        field: 'posts',
+        claimed: claimed.posts,
+        actual: actualPosts,
+      })
+    }
+  }
+
+  for (const thread of visibleThreads) {
+    const actual = Math.max(
+      0,
+      visiblePosts.filter((post) => post.legacyThreadId === thread.legacyId).length - 1,
+    )
+    if (thread.replyCount !== actual) {
+      differences.push({
+        legacyId: thread.legacyId,
+        field: 'replies',
+        claimed: thread.replyCount,
+        actual,
+      })
+    }
+  }
+
+  return differences
+}

package/src/source.ts

@@ -0,0 +1,98 @@
+import type {
+  ImportedAttachment,
+  ImportedAvatar,
+  ImportedBan,
+  ImportedForum,
+  ImportedPoll,
+  ImportedPollVote,
+  ImportedPost,
+  ImportedPrivateMessage,
+  ImportedReputation,
+  ImportedSubscription,
+  ImportedThread,
+  ImportedUser,
+  ImportedUserRelation,
+  ImportedWarning,
+} from './types'
+
+export interface Page<T> {
+  readonly rows: readonly T[]
+  readonly nextCursor: number | null
+}
+
+export type ImportSourceName = 'mybb' | 'phpbb'
+
+export interface ImportSource {
+  readonly name: ImportSourceName
+  users(afterId: number, limit: number): Promise<Page<ImportedUser>>
+  forums(afterId: number, limit: number): Promise<Page<ImportedForum>>
+  threads(afterId: number, limit: number): Promise<Page<ImportedThread>>
+  posts(afterId: number, limit: number): Promise<Page<ImportedPost>>
+  avatars(afterId: number, limit: number): Promise<Page<ImportedAvatar>>
+  attachments(afterId: number, limit: number): Promise<Page<ImportedAttachment>>
+  polls(afterId: number, limit: number): Promise<Page<ImportedPoll>>
+  pollVotes(afterId: number, limit: number): Promise<Page<ImportedPollVote>>
+  privateMessages(afterId: number, limit: number): Promise<Page<ImportedPrivateMessage>>
+  threadSubscriptions(afterId: number, limit: number): Promise<Page<ImportedSubscription>>
+  forumSubscriptions(afterId: number, limit: number): Promise<Page<ImportedSubscription>>
+  reputation(afterId: number, limit: number): Promise<Page<ImportedReputation>>
+  warnings(afterId: number, limit: number): Promise<Page<ImportedWarning>>
+  bans(afterId: number, limit: number): Promise<Page<ImportedBan>>
+  userRelations(afterId: number, limit: number): Promise<Page<ImportedUserRelation>>
+}
+
+export function pageById<T>(
+  all: readonly T[],
+  idOf: (row: T) => number,
+  afterId: number,
+  limit: number,
+): Page<T> {
+  const rows = [...all]
+    .sort((a, b) => idOf(a) - idOf(b))
+    .filter((row) => idOf(row) > afterId)
+    .slice(0, limit)
+
+  const last = rows.at(-1)
+  return { rows, nextCursor: rows.length < limit || last === undefined ? null : idOf(last) }
+}
+
+export async function pageByGroup<T>(input: {
+  readonly afterGroup: number
+  readonly limit: number
+  readonly groupOf: (row: T) => number
+  readonly fetch: (afterGroup: number, limit: number) => Promise<readonly T[]>
+  readonly fetchGroup: (group: number) => Promise<readonly T[]>
+}): Promise<Page<T>> {
+  const rows = await input.fetch(input.afterGroup, input.limit)
+  const last = rows.at(-1)
+  if (rows.length < input.limit || last === undefined) return { rows, nextCursor: null }
+
+  const lastGroup = input.groupOf(last)
+  const kept = rows.filter((row) => input.groupOf(row) !== lastGroup)
+
+  if (kept.length === 0) {
+    const whole = await input.fetchGroup(lastGroup)
+    return { rows: whole, nextCursor: lastGroup }
+  }
+
+  const lastKept = kept.at(-1)
+  return {
+    rows: kept,
+    nextCursor: lastKept === undefined ? null : input.groupOf(lastKept),
+  }
+}
+
+export function groupRows<T>(
+  all: readonly T[],
+  groupOf: (row: T) => number,
+): {
+  fetch: (afterGroup: number, limit: number) => Promise<readonly T[]>
+  fetchGroup: (group: number) => Promise<readonly T[]>
+} {
+  const sorted = [...all].sort((a, b) => groupOf(a) - groupOf(b))
+  return {
+    fetch: (afterGroup, limit) =>
+      Promise.resolve(sorted.filter((row) => groupOf(row) > afterGroup).slice(0, limit)),
+    fetchGroup: (group) => Promise.resolve(sorted.filter((row) => groupOf(row) === group)),
+  }
+}