@meith/import 0.16.0

package/src/map.ts

@@ -0,0 +1,297 @@
+import type {
+  MybbAttachment,
+  MybbAvatarRow,
+  MybbBan,
+  MybbForum,
+  MybbForumSubscription,
+  MybbPoll,
+  MybbPollVote,
+  MybbPost,
+  MybbPrivateMessage,
+  MybbRelationList,
+  MybbReputation,
+  MybbThread,
+  MybbThreadSubscription,
+  MybbUser,
+  MybbWarning,
+} from './mybb-rows'
+import type {
+  ImportedAttachment,
+  ImportedAvatar,
+  ImportedBan,
+  ImportedForum,
+  ImportedPoll,
+  ImportedPollVote,
+  ImportedPost,
+  ImportedPrivateMessage,
+  ImportedReputation,
+  ImportedSubscription,
+  ImportedThread,
+  ImportedUser,
+  ImportedUserRelation,
+  ImportedWarning,
+} from './types'
+import { fromUnixSeconds, visibilityOf } from './types'
+
+export function mapUser(row: MybbUser): ImportedUser {
+  return {
+    legacyId: row.uid,
+    username: row.username,
+    email: row.email.trim().toLowerCase(),
+    legacyPasswordHash: `mybb$${row.salt}$${row.password}`,
+    registeredAt: fromUnixSeconds(row.regdate) ?? new Date(0),
+    lastVisitAt: fromUnixSeconds(row.lastvisit),
+    postCount: Math.max(0, row.postnum),
+    legacyGroupId: row.usergroup,
+  }
+}
+
+const FORUM_TYPES: Readonly<Record<string, ImportedForum['type']>> = {
+  f: 'forum',
+  c: 'category',
+  l: 'link',
+}
+
+export function mapForum(row: MybbForum): ImportedForum {
+  return {
+    legacyId: row.fid,
+    type: FORUM_TYPES[row.type] ?? 'forum',
+    title: row.name,
+    description: row.description.trim() === '' ? null : row.description,
+    legacyParentId: row.pid === 0 ? null : row.pid,
+    displayOrder: row.disporder,
+    linkUrl: row.type === 'l' && row.linkto.trim() !== '' ? row.linkto : null,
+  }
+}
+
+export function mapThread(row: MybbThread): ImportedThread {
+  const createdAt = fromUnixSeconds(row.dateline) ?? new Date(0)
+
+  return {
+    legacyId: row.tid,
+    legacyForumId: row.fid,
+    title: row.subject,
+    legacyAuthorId: row.uid,
+    authorUsername: row.username,
+    createdAt,
+    lastPostAt: fromUnixSeconds(row.lastpost) ?? createdAt,
+    replyCount: Math.max(0, row.replies),
+    viewCount: Math.max(0, row.views),
+    isSticky: row.sticky === 1,
+    isLocked: row.closed === '1',
+    visibility: visibilityOf(row.visible),
+  }
+}
+
+export function mapPost(row: MybbPost): ImportedPost {
+  return {
+    legacyId: row.pid,
+    legacyThreadId: row.tid,
+    legacyForumId: row.fid,
+    legacyAuthorId: row.uid,
+    authorUsername: row.username,
+    body: row.message,
+    createdAt: fromUnixSeconds(row.dateline) ?? new Date(0),
+    editedAt: row.edituid === 0 ? null : fromUnixSeconds(row.edittime),
+    visibility: visibilityOf(row.visible),
+  }
+}
+
+const AVATAR_SOURCES: Readonly<Record<string, ImportedAvatar['source']>> = {
+  upload: 'upload',
+  gallery: 'gallery',
+  remote: 'remote',
+}
+
+export function mapAvatar(row: MybbAvatarRow): ImportedAvatar | null {
+  const source = AVATAR_SOURCES[row.avatartype]
+  if (source === undefined) return null
+
+  const path = row.avatar.split('?')[0]?.replace(/^\.\//, '') ?? ''
+  if (path === '') return null
+
+  const [width, height] = row.avatardimensions.split('|').map((part) => Number(part))
+
+  return {
+    legacyUserId: row.uid,
+    source,
+    path,
+    width: Number.isFinite(width) && width! > 0 ? width! : null,
+    height: Number.isFinite(height) && height! > 0 ? height! : null,
+  }
+}
+
+export function mapAttachment(row: MybbAttachment): ImportedAttachment {
+  const path = row.attachname.replace(/^\.\//, '')
+  const thumbnail = row.thumbnail.trim()
+
+  return {
+    legacyId: row.aid,
+    legacyPostId: row.pid,
+    legacyUserId: row.uid,
+    filename: row.filename,
+    contentType: row.filetype.trim() === '' ? null : row.filetype.trim(),
+    sizeBytes: Math.max(0, row.filesize),
+    path,
+    thumbnailPath:
+      thumbnail === '' || thumbnail === 'SMALL' ? null : thumbnail.replace(/^\.\//, ''),
+    downloadCount: Math.max(0, row.downloads),
+    createdAt: fromUnixSeconds(row.dateuploaded) ?? new Date(0),
+  }
+}
+
+const OPTION_SEPARATOR = '||~|~||'
+
+export function mapPoll(row: MybbPoll): ImportedPoll | null {
+  const labels = row.options.split(OPTION_SEPARATOR)
+  const counts = row.votes.split(OPTION_SEPARATOR)
+  const createdAt = fromUnixSeconds(row.dateline) ?? new Date(0)
+
+  const options = labels
+    .map((label, index) => ({
+      order: index + 1,
+      label: label.trim(),
+      voteCount: Math.max(0, Number(counts[index]) || 0),
+    }))
+    .filter((option) => option.label !== '')
+
+  if (options.length === 0) return null
+
+  return {
+    legacyId: row.pid,
+    legacyThreadId: row.tid,
+    question: row.question,
+    options,
+    closesAt:
+      row.timeout > 0 ? new Date(createdAt.getTime() + row.timeout * 24 * 60 * 60 * 1000) : null,
+    createdAt,
+    maxOptions: row.multiple > 0 ? 0 : 1,
+    allowRevote: false,
+    publicVotes: row.public > 0,
+  }
+}
+
+export function mapPollVote(row: MybbPollVote): ImportedPollVote | null {
+  if (row.voteoption <= 0) return null
+  return {
+    legacyPollId: row.pid,
+    legacyUserId: row.uid,
+    optionOrder: row.voteoption,
+    votedAt: fromUnixSeconds(row.dateline),
+  }
+}
+
+const PM_FOLDERS: Readonly<Record<number, 'inbox' | 'sent' | 'trash'>> = {
+  1: 'inbox',
+  2: 'sent',
+  4: 'trash',
+}
+
+export function mapPrivateMessage(row: MybbPrivateMessage): ImportedPrivateMessage | null {
+  if (row.folder === 3) return null
+
+  return {
+    legacyId: row.pmid,
+    legacyAuthorId: row.fromid,
+    subject: row.subject,
+    body: row.message,
+    sentAt: fromUnixSeconds(row.dateline) ?? new Date(0),
+    copies: [
+      {
+        legacyOwnerId: row.uid,
+        folder: PM_FOLDERS[row.folder] ?? 'inbox',
+        readAt: row.status === 0 ? null : fromUnixSeconds(row.readtime),
+      },
+    ],
+  }
+}
+
+export function mapThreadSubscription(row: MybbThreadSubscription): ImportedSubscription {
+  return {
+    legacyId: row.sid,
+    legacyUserId: row.uid,
+    legacyTargetId: row.tid,
+    mode: row.notification === 0 ? 'none' : 'instant',
+  }
+}
+
+export function mapForumSubscription(row: MybbForumSubscription): ImportedSubscription {
+  return {
+    legacyId: row.uid,
+    legacyUserId: row.uid,
+    legacyTargetId: row.fid,
+    mode: 'instant',
+  }
+}
+
+export function mapReputation(row: MybbReputation): ImportedReputation {
+  return {
+    legacyId: row.rid,
+    legacyUserId: row.uid,
+    legacyGivenByUserId: row.adduid,
+    legacyPostId: row.pid === 0 ? null : row.pid,
+    points: row.reputation,
+    comment: row.comments,
+    createdAt: fromUnixSeconds(row.dateline) ?? new Date(0),
+  }
+}
+
+export function mapWarning(row: MybbWarning): ImportedWarning {
+  return {
+    legacyId: row.wid,
+    legacyUserId: row.uid,
+    legacyIssuedByUserId: row.issuedby,
+    legacyPostId: row.pid === 0 ? null : row.pid,
+    title: row.title,
+    points: Math.max(0, row.points),
+    notes: row.notes,
+    issuedAt: fromUnixSeconds(row.dateline) ?? new Date(0),
+    expiresAt: fromUnixSeconds(row.expires),
+    revokedAt: fromUnixSeconds(row.daterevoked),
+    revokeReason: row.revokereason.trim() === '' ? null : row.revokereason,
+  }
+}
+
+export function mapBan(row: MybbBan): ImportedBan {
+  return {
+    legacyId: row.uid,
+    legacyUserId: row.uid,
+    legacyBannedByUserId: row.admin,
+    reason: row.reason,
+    publicReason: null,
+    createdAt: fromUnixSeconds(row.dateline) ?? new Date(0),
+    expiresAt: fromUnixSeconds(row.lifted),
+  }
+}
+
+function idList(list: string): number[] {
+  return [
+    ...new Set(
+      list
+        .split(',')
+        .map((part) => Number(part.trim()))
+        .filter((id) => Number.isSafeInteger(id) && id > 0),
+    ),
+  ]
+}
+
+export function mapRelationList(row: MybbRelationList): ImportedUserRelation[] {
+  const buddies = idList(row.buddylist)
+  const ignoredSet = new Set(idList(row.ignorelist))
+  const buddySet = new Set(buddies)
+
+  return [
+    ...buddies.map((other) => ({
+      legacyUserId: row.uid,
+      legacyOtherUserId: other,
+      kind: 'buddy' as const,
+    })),
+    ...[...ignoredSet]
+      .filter((other) => !buddySet.has(other))
+      .map((other) => ({
+        legacyUserId: row.uid,
+        legacyOtherUserId: other,
+        kind: 'ignore' as const,
+      })),
+  ].filter((relation) => relation.legacyOtherUserId !== row.uid)
+}

package/src/memory-sink.fixture.ts

@@ -0,0 +1,64 @@
+import type { ImportSink, WriteResult } from './run'
+
+const byLegacyId = (row: { legacyId: number }): string => String(row.legacyId)
+const byPair = (row: { legacyUserId: number; legacyTargetId: number }): string =>
+  `${row.legacyUserId}:${row.legacyTargetId}`
+
+export class MemorySink implements ImportSink {
+  readonly stored = new Map<string, Map<string, unknown>>()
+  readonly calls: string[] = []
+
+  putUsers = (rows: readonly { legacyId: number }[]) => this.#put('users', rows, byLegacyId)
+  putForums = (rows: readonly { legacyId: number }[]) => this.#put('forums', rows, byLegacyId)
+  putThreads = (rows: readonly { legacyId: number }[]) => this.#put('threads', rows, byLegacyId)
+  putPosts = (rows: readonly { legacyId: number }[]) => this.#put('posts', rows, byLegacyId)
+  putAvatars = (rows: readonly { legacyUserId: number }[]) =>
+    this.#put('avatars', rows, (row) => String(row.legacyUserId))
+  putAttachments = (rows: readonly { legacyId: number }[]) =>
+    this.#put('attachments', rows, byLegacyId)
+  putPolls = (rows: readonly { legacyId: number }[]) => this.#put('polls', rows, byLegacyId)
+  putPollVotes = (
+    rows: readonly { legacyPollId: number; legacyUserId: number; optionOrder: number }[],
+  ) =>
+    this.#put(
+      'pollVotes',
+      rows,
+      (row) => `${row.legacyPollId}:${row.legacyUserId}:${row.optionOrder}`,
+    )
+  putPrivateMessages = (rows: readonly { legacyId: number }[]) =>
+    this.#put('privateMessages', rows, byLegacyId)
+  putThreadSubscriptions = (rows: readonly { legacyUserId: number; legacyTargetId: number }[]) =>
+    this.#put('threadSubscriptions', rows, byPair)
+  putForumSubscriptions = (rows: readonly { legacyUserId: number; legacyTargetId: number }[]) =>
+    this.#put('forumSubscriptions', rows, byPair)
+  putReputation = (rows: readonly { legacyId: number }[]) =>
+    this.#put('reputation', rows, byLegacyId)
+  putWarnings = (rows: readonly { legacyId: number }[]) => this.#put('warnings', rows, byLegacyId)
+  putBans = (rows: readonly { legacyId: number }[]) => this.#put('bans', rows, byLegacyId)
+  putUserRelations = (rows: readonly { legacyUserId: number; legacyOtherUserId: number }[]) =>
+    this.#put('userRelations', rows, (row) => `${row.legacyUserId}:${row.legacyOtherUserId}`)
+
+  count(kind: string): number {
+    return this.stored.get(kind)?.size ?? 0
+  }
+
+  rows<T>(kind: string): readonly T[] {
+    return [...(this.stored.get(kind)?.values() ?? [])] as T[]
+  }
+
+  async #put<T>(kind: string, rows: readonly T[], keyOf: (row: T) => string): Promise<WriteResult> {
+    this.calls.push(`${kind}:${rows.length}`)
+    const table = this.stored.get(kind) ?? new Map<string, unknown>()
+    this.stored.set(kind, table)
+
+    let inserted = 0
+    let updated = 0
+    for (const row of rows) {
+      const key = keyOf(row)
+      if (table.has(key)) updated += 1
+      else inserted += 1
+      table.set(key, row)
+    }
+    return { inserted, updated, skipped: [] }
+  }
+}

package/src/mybb-fixture.ts

@@ -0,0 +1,185 @@
+import {
+  mapAttachment,
+  mapAvatar,
+  mapBan,
+  mapForum,
+  mapForumSubscription,
+  mapPoll,
+  mapPollVote,
+  mapPost,
+  mapPrivateMessage,
+  mapRelationList,
+  mapReputation,
+  mapThread,
+  mapThreadSubscription,
+  mapUser,
+  mapWarning,
+} from './map'
+import type { MybbTables } from './mybb-rows'
+import { groupRows, type ImportSource, type Page, pageByGroup, pageById } from './source'
+import type {
+  ImportedAttachment,
+  ImportedAvatar,
+  ImportedBan,
+  ImportedForum,
+  ImportedPoll,
+  ImportedPollVote,
+  ImportedPost,
+  ImportedPrivateMessage,
+  ImportedReputation,
+  ImportedSubscription,
+  ImportedThread,
+  ImportedUser,
+  ImportedUserRelation,
+  ImportedWarning,
+} from './types'
+
+function mapPage<Raw, Mapped>(page: Page<Raw>, map: (row: Raw) => Mapped | null): Page<Mapped> {
+  return {
+    rows: page.rows.map(map).filter((row): row is NonNullable<Mapped> => row !== null),
+    nextCursor: page.nextCursor,
+  }
+}
+
+export class FixtureMybbSource implements ImportSource {
+  readonly name = 'mybb'
+
+  constructor(private readonly data: MybbTables) {}
+
+  users(afterId: number, limit: number): Promise<Page<ImportedUser>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.users ?? [], (row) => row.uid, afterId, limit),
+        mapUser,
+      ),
+    )
+  }
+
+  forums(afterId: number, limit: number): Promise<Page<ImportedForum>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.forums ?? [], (row) => row.fid, afterId, limit),
+        mapForum,
+      ),
+    )
+  }
+
+  threads(afterId: number, limit: number): Promise<Page<ImportedThread>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.threads ?? [], (row) => row.tid, afterId, limit),
+        mapThread,
+      ),
+    )
+  }
+
+  posts(afterId: number, limit: number): Promise<Page<ImportedPost>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.posts ?? [], (row) => row.pid, afterId, limit),
+        mapPost,
+      ),
+    )
+  }
+
+  avatars(afterId: number, limit: number): Promise<Page<ImportedAvatar>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.avatars ?? [], (row) => row.uid, afterId, limit),
+        mapAvatar,
+      ),
+    )
+  }
+
+  attachments(afterId: number, limit: number): Promise<Page<ImportedAttachment>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.attachments ?? [], (row) => row.aid, afterId, limit),
+        mapAttachment,
+      ),
+    )
+  }
+
+  polls(afterId: number, limit: number): Promise<Page<ImportedPoll>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.polls ?? [], (row) => row.pid, afterId, limit),
+        mapPoll,
+      ),
+    )
+  }
+
+  pollVotes(afterId: number, limit: number): Promise<Page<ImportedPollVote>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.pollVotes ?? [], (row) => row.vid, afterId, limit),
+        mapPollVote,
+      ),
+    )
+  }
+
+  privateMessages(afterId: number, limit: number): Promise<Page<ImportedPrivateMessage>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.privateMessages ?? [], (row) => row.pmid, afterId, limit),
+        mapPrivateMessage,
+      ),
+    )
+  }
+
+  threadSubscriptions(afterId: number, limit: number): Promise<Page<ImportedSubscription>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.threadSubscriptions ?? [], (row) => row.sid, afterId, limit),
+        mapThreadSubscription,
+      ),
+    )
+  }
+
+  async forumSubscriptions(afterId: number, limit: number): Promise<Page<ImportedSubscription>> {
+    const { fetch, fetchGroup } = groupRows(this.data.forumSubscriptions ?? [], (row) => row.uid)
+    const page = await pageByGroup({
+      afterGroup: afterId,
+      limit,
+      groupOf: (row) => row.uid,
+      fetch,
+      fetchGroup,
+    })
+    return mapPage(page, mapForumSubscription)
+  }
+
+  reputation(afterId: number, limit: number): Promise<Page<ImportedReputation>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.reputation ?? [], (row) => row.rid, afterId, limit),
+        mapReputation,
+      ),
+    )
+  }
+
+  warnings(afterId: number, limit: number): Promise<Page<ImportedWarning>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.warnings ?? [], (row) => row.wid, afterId, limit),
+        mapWarning,
+      ),
+    )
+  }
+
+  bans(afterId: number, limit: number): Promise<Page<ImportedBan>> {
+    return Promise.resolve(
+      mapPage(
+        pageById(this.data.bans ?? [], (row) => row.uid, afterId, limit),
+        mapBan,
+      ),
+    )
+  }
+
+  userRelations(afterId: number, limit: number): Promise<Page<ImportedUserRelation>> {
+    const page = pageById(this.data.relationLists ?? [], (row) => row.uid, afterId, limit)
+    return Promise.resolve({
+      rows: page.rows.flatMap(mapRelationList),
+      nextCursor: page.nextCursor,
+    })
+  }
+}

package/src/mybb-rows.ts

@@ -0,0 +1,172 @@
+export interface MybbUser {
+  readonly uid: number
+  readonly username: string
+  readonly email: string
+  readonly password: string
+  readonly salt: string
+  readonly usergroup: number
+  readonly regdate: number
+  readonly lastvisit: number
+  readonly postnum: number
+}
+
+export interface MybbForum {
+  readonly fid: number
+  readonly name: string
+  readonly description: string
+  readonly type: string
+  readonly pid: number
+  readonly disporder: number
+  readonly linkto: string
+  readonly threads: number
+  readonly posts: number
+}
+
+export interface MybbThread {
+  readonly tid: number
+  readonly fid: number
+  readonly subject: string
+  readonly uid: number
+  readonly username: string
+  readonly dateline: number
+  readonly lastpost: number
+  readonly replies: number
+  readonly views: number
+  readonly sticky: number
+  readonly closed: string
+  readonly visible: number
+}
+
+export interface MybbPost {
+  readonly pid: number
+  readonly tid: number
+  readonly fid: number
+  readonly uid: number
+  readonly username: string
+  readonly subject: string
+  readonly message: string
+  readonly dateline: number
+  readonly edituid: number
+  readonly edittime: number
+  readonly visible: number
+}
+
+export interface MybbAvatarRow {
+  readonly uid: number
+  readonly avatar: string
+  readonly avatartype: string
+  readonly avatardimensions: string
+}
+
+export interface MybbAttachment {
+  readonly aid: number
+  readonly pid: number
+  readonly uid: number
+  readonly filename: string
+  readonly filetype: string
+  readonly filesize: number
+  readonly attachname: string
+  readonly thumbnail: string
+  readonly downloads: number
+  readonly dateuploaded: number
+}
+
+export interface MybbPoll {
+  readonly pid: number
+  readonly tid: number
+  readonly question: string
+  readonly dateline: number
+  readonly options: string
+  readonly votes: string
+  readonly timeout: number
+  readonly multiple: number
+  readonly public: number
+}
+
+export interface MybbPollVote {
+  readonly vid: number
+  readonly pid: number
+  readonly uid: number
+  readonly voteoption: number
+  readonly dateline: number
+}
+
+export interface MybbPrivateMessage {
+  readonly pmid: number
+  readonly uid: number
+  readonly fromid: number
+  readonly subject: string
+  readonly message: string
+  readonly dateline: number
+  readonly folder: number
+  readonly status: number
+  readonly readtime: number
+}
+
+export interface MybbThreadSubscription {
+  readonly sid: number
+  readonly uid: number
+  readonly tid: number
+  readonly notification: number
+}
+
+export interface MybbForumSubscription {
+  readonly fid: number
+  readonly uid: number
+}
+
+export interface MybbReputation {
+  readonly rid: number
+  readonly uid: number
+  readonly adduid: number
+  readonly pid: number
+  readonly reputation: number
+  readonly dateline: number
+  readonly comments: string
+}
+
+export interface MybbWarning {
+  readonly wid: number
+  readonly uid: number
+  readonly pid: number
+  readonly title: string
+  readonly points: number
+  readonly dateline: number
+  readonly issuedby: number
+  readonly expires: number
+  readonly daterevoked: number
+  readonly revokereason: string
+  readonly notes: string
+}
+
+export interface MybbBan {
+  readonly uid: number
+  readonly admin: number
+  readonly dateline: number
+  readonly lifted: number
+  readonly reason: string
+}
+
+export interface MybbRelationList {
+  readonly uid: number
+  readonly buddylist: string
+  readonly ignorelist: string
+}
+
+export interface MybbTables {
+  readonly users?: readonly MybbUser[]
+  readonly forums?: readonly MybbForum[]
+  readonly threads?: readonly MybbThread[]
+  readonly posts?: readonly MybbPost[]
+  readonly avatars?: readonly MybbAvatarRow[]
+  readonly attachments?: readonly MybbAttachment[]
+  readonly polls?: readonly MybbPoll[]
+  readonly pollVotes?: readonly MybbPollVote[]
+  readonly privateMessages?: readonly MybbPrivateMessage[]
+  readonly threadSubscriptions?: readonly MybbThreadSubscription[]
+  readonly forumSubscriptions?: readonly MybbForumSubscription[]
+  readonly reputation?: readonly MybbReputation[]
+  readonly warnings?: readonly MybbWarning[]
+  readonly bans?: readonly MybbBan[]
+  readonly relationLists?: readonly MybbRelationList[]
+}