@meith/import 0.16.0

package/src/phpbb-map.ts

@@ -0,0 +1,297 @@
+import type {
+  PhpbbAttachment,
+  PhpbbAvatarRow,
+  PhpbbBan,
+  PhpbbForum,
+  PhpbbPollOption,
+  PhpbbPollTopic,
+  PhpbbPollVote,
+  PhpbbPost,
+  PhpbbPrivateMessage,
+  PhpbbPrivateMessageCopy,
+  PhpbbTopic,
+  PhpbbUser,
+  PhpbbWarning,
+  PhpbbWatch,
+  PhpbbZebra,
+} from './phpbb-rows'
+import { decodeHtmlEntities, decodePhpbbText } from './phpbb-text'
+import type {
+  ImportedAttachment,
+  ImportedAvatar,
+  ImportedBan,
+  ImportedForum,
+  ImportedPoll,
+  ImportedPollVote,
+  ImportedPost,
+  ImportedPrivateMessage,
+  ImportedReputation,
+  ImportedSubscription,
+  ImportedThread,
+  ImportedUser,
+  ImportedUserRelation,
+  ImportedWarning,
+  Visibility,
+} from './types'
+import { fromUnixSeconds } from './types'
+
+const BOT_USER_TYPE = 2
+
+export function mapPhpbbUser(row: PhpbbUser): ImportedUser | null {
+  if (row.user_type === BOT_USER_TYPE) return null
+
+  return {
+    legacyId: row.user_id,
+    username: decodeHtmlEntities(row.username),
+    email: row.user_email.trim().toLowerCase(),
+    legacyPasswordHash: `phpbb$${row.user_password}`,
+    registeredAt: fromUnixSeconds(row.user_regdate) ?? new Date(0),
+    lastVisitAt: fromUnixSeconds(row.user_lastvisit),
+    postCount: Math.max(0, row.user_posts),
+    legacyGroupId: row.group_id,
+  }
+}
+
+const FORUM_TYPES: Readonly<Record<number, ImportedForum['type']>> = {
+  0: 'category',
+  1: 'forum',
+  2: 'link',
+}
+
+export function mapPhpbbForum(row: PhpbbForum): ImportedForum {
+  const description = decodeHtmlEntities(row.forum_desc).trim()
+
+  return {
+    legacyId: row.forum_id,
+    type: FORUM_TYPES[row.forum_type] ?? 'forum',
+    title: decodeHtmlEntities(row.forum_name),
+    description: description === '' ? null : description,
+    legacyParentId: row.parent_id === 0 ? null : row.parent_id,
+    displayOrder: row.left_id,
+    linkUrl: row.forum_type === 2 && row.forum_link.trim() !== '' ? row.forum_link : null,
+  }
+}
+
+export function phpbbVisibility(visibility: number): Visibility {
+  if (visibility === 1) return 'visible'
+  if (visibility === 2) return 'deleted'
+  return 'unapproved'
+}
+
+export function mapPhpbbTopic(row: PhpbbTopic): ImportedThread | null {
+  if (row.topic_moved_id !== 0) return null
+
+  const createdAt = fromUnixSeconds(row.topic_time) ?? new Date(0)
+
+  return {
+    legacyId: row.topic_id,
+    legacyForumId: row.forum_id,
+    title: decodeHtmlEntities(row.topic_title),
+    legacyAuthorId: row.topic_poster,
+    authorUsername: decodeHtmlEntities(row.topic_first_poster_name),
+    createdAt,
+    lastPostAt: fromUnixSeconds(row.topic_last_post_time) ?? createdAt,
+    replyCount: Math.max(0, row.topic_posts_approved - 1),
+    viewCount: Math.max(0, row.topic_views),
+    isSticky: row.topic_type > 0,
+    isLocked: row.topic_status === 1,
+    visibility: phpbbVisibility(row.topic_visibility),
+  }
+}
+
+export function mapPhpbbPost(row: PhpbbPost): ImportedPost {
+  return {
+    legacyId: row.post_id,
+    legacyThreadId: row.topic_id,
+    legacyForumId: row.forum_id,
+    legacyAuthorId: row.poster_id,
+    authorUsername: decodeHtmlEntities(row.post_username),
+    body: decodePhpbbText(row.post_text, row.bbcode_uid),
+    createdAt: fromUnixSeconds(row.post_time) ?? new Date(0),
+    editedAt: fromUnixSeconds(row.post_edit_time),
+    visibility: phpbbVisibility(row.post_visibility),
+  }
+}
+
+const AVATAR_SOURCES: Readonly<Record<string, ImportedAvatar['source']>> = {
+  'avatar.driver.upload': 'upload',
+  'avatar.driver.remote': 'remote',
+  'avatar.driver.gallery': 'gallery',
+  '1': 'upload',
+  '2': 'remote',
+  '3': 'gallery',
+}
+
+export function mapPhpbbAvatar(row: PhpbbAvatarRow): ImportedAvatar | null {
+  const source = AVATAR_SOURCES[row.user_avatar_type.trim()]
+  if (source === undefined) return null
+
+  const value = row.user_avatar.split('?')[0]?.trim() ?? ''
+  if (value === '') return null
+
+  const path =
+    source === 'upload'
+      ? `images/avatars/upload/*_${row.user_id}${extensionOf(value)}`
+      : source === 'gallery'
+        ? `images/avatars/gallery/${value}`
+        : value
+
+  return {
+    legacyUserId: row.user_id,
+    source,
+    path,
+    width: row.user_avatar_width > 0 ? row.user_avatar_width : null,
+    height: row.user_avatar_height > 0 ? row.user_avatar_height : null,
+  }
+}
+
+function extensionOf(filename: string): string {
+  const dot = filename.lastIndexOf('.')
+  return dot === -1 ? '' : filename.slice(dot).toLowerCase()
+}
+
+export function mapPhpbbAttachment(row: PhpbbAttachment): ImportedAttachment {
+  const mimetype = row.mimetype.trim()
+
+  return {
+    legacyId: row.attach_id,
+    legacyPostId: row.post_msg_id,
+    legacyUserId: row.poster_id,
+    filename: decodeHtmlEntities(row.real_filename),
+    contentType: mimetype === '' ? null : mimetype,
+    sizeBytes: Math.max(0, row.filesize),
+    path: `files/${row.physical_filename}`,
+    thumbnailPath: row.thumbnail === 1 ? `files/thumb_${row.physical_filename}` : null,
+    downloadCount: Math.max(0, row.download_count),
+    createdAt: fromUnixSeconds(row.filetime) ?? new Date(0),
+  }
+}
+
+export function mapPhpbbPoll(
+  topic: PhpbbPollTopic,
+  options: readonly PhpbbPollOption[],
+): ImportedPoll | null {
+  const createdAt = fromUnixSeconds(topic.poll_start) ?? new Date(0)
+
+  const mapped = options
+    .filter((option) => option.topic_id === topic.topic_id)
+    .sort((a, b) => a.poll_option_id - b.poll_option_id)
+    .map((option) => ({
+      order: option.poll_option_id,
+      label: decodeHtmlEntities(option.poll_option_text).trim(),
+      voteCount: Math.max(0, option.poll_option_total),
+    }))
+    .filter((option) => option.label !== '')
+
+  if (mapped.length === 0) return null
+
+  return {
+    legacyId: topic.topic_id,
+    legacyThreadId: topic.topic_id,
+    question: decodeHtmlEntities(topic.poll_title),
+    options: mapped,
+    closesAt:
+      topic.poll_length > 0 ? new Date((topic.poll_start + topic.poll_length) * 1000) : null,
+    createdAt,
+    maxOptions: Math.min(Math.max(1, topic.poll_max_options), mapped.length),
+    allowRevote: topic.poll_vote_change > 0,
+    publicVotes: false,
+  }
+}
+
+export function mapPhpbbPollVote(row: PhpbbPollVote): ImportedPollVote {
+  return {
+    legacyPollId: row.topic_id,
+    legacyUserId: row.vote_user_id,
+    optionOrder: row.poll_option_id,
+    votedAt: null,
+  }
+}
+
+export function mapPhpbbPrivateMessage(
+  message: PhpbbPrivateMessage,
+  copies: readonly PhpbbPrivateMessageCopy[],
+): ImportedPrivateMessage | null {
+  const mapped = copies
+    .filter((copy) => copy.msg_id === message.msg_id)
+    .map((copy) => ({
+      legacyOwnerId: copy.user_id,
+      folder:
+        copy.pm_deleted === 1
+          ? ('trash' as const)
+          : copy.user_id === message.author_id && copy.folder_id < 0
+            ? ('sent' as const)
+            : ('inbox' as const),
+      readAt: copy.pm_unread === 1 ? null : (fromUnixSeconds(message.message_time) ?? null),
+    }))
+
+  if (mapped.length === 0) return null
+
+  return {
+    legacyId: message.msg_id,
+    legacyAuthorId: message.author_id,
+    subject: decodeHtmlEntities(message.message_subject),
+    body: decodePhpbbText(message.message_text, message.bbcode_uid),
+    sentAt: fromUnixSeconds(message.message_time) ?? new Date(0),
+    copies: mapped,
+  }
+}
+
+export function mapPhpbbWatch(row: PhpbbWatch): ImportedSubscription {
+  return {
+    legacyId: row.user_id,
+    legacyUserId: row.user_id,
+    legacyTargetId: row.target_id,
+    mode: 'instant',
+  }
+}
+
+export function mapPhpbbWarning(row: PhpbbWarning): ImportedWarning {
+  return {
+    legacyId: row.warning_id,
+    legacyUserId: row.user_id,
+    legacyIssuedByUserId: 0,
+    legacyPostId: row.post_id === 0 ? null : row.post_id,
+    title: 'Warning',
+    points: 1,
+    notes: '',
+    issuedAt: fromUnixSeconds(row.warning_time) ?? new Date(0),
+    expiresAt: null,
+    revokedAt: null,
+    revokeReason: null,
+  }
+}
+
+export function mapPhpbbBan(row: PhpbbBan): ImportedBan | null {
+  if (row.ban_userid === 0 || row.ban_exclude === 1) return null
+
+  const publicReason = decodeHtmlEntities(row.ban_give_reason).trim()
+
+  return {
+    legacyId: row.ban_id,
+    legacyUserId: row.ban_userid,
+    legacyBannedByUserId: 0,
+    reason: decodeHtmlEntities(row.ban_reason),
+    publicReason: publicReason === '' ? null : publicReason,
+    createdAt: fromUnixSeconds(row.ban_start) ?? new Date(0),
+    expiresAt: fromUnixSeconds(row.ban_end),
+  }
+}
+
+export function mapPhpbbZebra(row: PhpbbZebra): ImportedUserRelation | null {
+  if (row.user_id === row.zebra_id) return null
+  if (row.friend === 1) {
+    return { legacyUserId: row.user_id, legacyOtherUserId: row.zebra_id, kind: 'buddy' }
+  }
+  if (row.foe === 1) {
+    return { legacyUserId: row.user_id, legacyOtherUserId: row.zebra_id, kind: 'ignore' }
+  }
+  return null
+}
+
+export function noPhpbbReputation(): Promise<{
+  rows: readonly ImportedReputation[]
+  nextCursor: null
+}> {
+  return Promise.resolve({ rows: [], nextCursor: null })
+}

package/src/phpbb-mysql-source.ts

@@ -0,0 +1,355 @@
+import {
+  assertSafePrefix,
+  connectMysql,
+  keysetPage,
+  type MysqlSourceOptions,
+  mapPage,
+  type Queryable,
+  selectRows,
+} from './mysql-source'
+import {
+  mapPhpbbAttachment,
+  mapPhpbbAvatar,
+  mapPhpbbBan,
+  mapPhpbbForum,
+  mapPhpbbPoll,
+  mapPhpbbPollVote,
+  mapPhpbbPost,
+  mapPhpbbPrivateMessage,
+  mapPhpbbTopic,
+  mapPhpbbUser,
+  mapPhpbbWarning,
+  mapPhpbbWatch,
+  mapPhpbbZebra,
+  noPhpbbReputation,
+} from './phpbb-map'
+import type {
+  PhpbbAttachment,
+  PhpbbAvatarRow,
+  PhpbbBan,
+  PhpbbForum,
+  PhpbbPollOption,
+  PhpbbPollTopic,
+  PhpbbPollVote,
+  PhpbbPost,
+  PhpbbPrivateMessage,
+  PhpbbPrivateMessageCopy,
+  PhpbbTopic,
+  PhpbbUser,
+  PhpbbWarning,
+  PhpbbWatch,
+  PhpbbZebra,
+} from './phpbb-rows'
+import { type ImportSource, type Page, pageByGroup } from './source'
+import type {
+  ImportedAttachment,
+  ImportedAvatar,
+  ImportedBan,
+  ImportedForum,
+  ImportedPoll,
+  ImportedPollVote,
+  ImportedPost,
+  ImportedPrivateMessage,
+  ImportedReputation,
+  ImportedSubscription,
+  ImportedThread,
+  ImportedUser,
+  ImportedUserRelation,
+  ImportedWarning,
+} from './types'
+
+export class MysqlPhpbbSource implements ImportSource {
+  readonly name = 'phpbb'
+
+  private constructor(
+    private readonly connection: Queryable,
+    private readonly prefix: string,
+  ) {}
+
+  static async connect(options: MysqlSourceOptions): Promise<MysqlPhpbbSource> {
+    const prefix = options.tablePrefix ?? 'phpbb_'
+    assertSafePrefix(prefix)
+    return new MysqlPhpbbSource(await connectMysql(options), prefix)
+  }
+
+  async close(): Promise<void> {
+    await this.connection.end()
+  }
+
+  async users(afterId: number, limit: number): Promise<Page<ImportedUser>> {
+    return mapPage(
+      await keysetPage<PhpbbUser>(
+        this.connection,
+        `select user_id, user_type, username, user_email, user_password,
+                user_regdate, user_lastvisit, user_posts, group_id
+           from \`${this.prefix}users\``,
+        'user_id',
+        (row) => row.user_id,
+        afterId,
+        limit,
+      ),
+      mapPhpbbUser,
+    )
+  }
+
+  async forums(afterId: number, limit: number): Promise<Page<ImportedForum>> {
+    return mapPage(
+      await keysetPage<PhpbbForum>(
+        this.connection,
+        `select forum_id, parent_id, forum_name, forum_desc, forum_type, forum_link, left_id
+           from \`${this.prefix}forums\``,
+        'forum_id',
+        (row) => row.forum_id,
+        afterId,
+        limit,
+      ),
+      mapPhpbbForum,
+    )
+  }
+
+  async threads(afterId: number, limit: number): Promise<Page<ImportedThread>> {
+    return mapPage(
+      await keysetPage<PhpbbTopic>(
+        this.connection,
+        `select topic_id, forum_id, topic_title, topic_poster, topic_first_poster_name,
+                topic_time, topic_last_post_time, topic_views, topic_posts_approved,
+                topic_status, topic_type, topic_visibility, topic_moved_id
+           from \`${this.prefix}topics\``,
+        'topic_id',
+        (row) => row.topic_id,
+        afterId,
+        limit,
+      ),
+      mapPhpbbTopic,
+    )
+  }
+
+  async posts(afterId: number, limit: number): Promise<Page<ImportedPost>> {
+    return mapPage(
+      await keysetPage<PhpbbPost>(
+        this.connection,
+        `select post_id, topic_id, forum_id, poster_id, post_username, post_subject,
+                post_text, bbcode_uid, post_time, post_edit_time, post_visibility
+           from \`${this.prefix}posts\``,
+        'post_id',
+        (row) => row.post_id,
+        afterId,
+        limit,
+      ),
+      mapPhpbbPost,
+    )
+  }
+
+  async avatars(afterId: number, limit: number): Promise<Page<ImportedAvatar>> {
+    const rows = await selectRows<PhpbbAvatarRow>(
+      this.connection,
+      `select user_id, user_avatar, user_avatar_type, user_avatar_width, user_avatar_height
+         from \`${this.prefix}users\`
+        where user_id > ? and user_avatar <> ''
+        order by user_id asc limit ?`,
+      [afterId, limit],
+    )
+    const last = rows.at(-1)
+    return mapPage(
+      { rows, nextCursor: rows.length < limit || last === undefined ? null : last.user_id },
+      mapPhpbbAvatar,
+    )
+  }
+
+  async attachments(afterId: number, limit: number): Promise<Page<ImportedAttachment>> {
+    const rows = await selectRows<PhpbbAttachment>(
+      this.connection,
+      `select attach_id, post_msg_id, poster_id, real_filename, physical_filename,
+              mimetype, filesize, download_count, filetime, thumbnail
+         from \`${this.prefix}attachments\`
+        where attach_id > ? and in_message = 0
+        order by attach_id asc limit ?`,
+      [afterId, limit],
+    )
+    const last = rows.at(-1)
+    return mapPage(
+      { rows, nextCursor: rows.length < limit || last === undefined ? null : last.attach_id },
+      mapPhpbbAttachment,
+    )
+  }
+
+  async polls(afterId: number, limit: number): Promise<Page<ImportedPoll>> {
+    const topics = await selectRows<PhpbbPollTopic>(
+      this.connection,
+      `select topic_id, poll_title, poll_start, poll_length, poll_max_options, poll_vote_change
+         from \`${this.prefix}topics\`
+        where topic_id > ? and poll_title <> '' and topic_moved_id = 0
+        order by topic_id asc limit ?`,
+      [afterId, limit],
+    )
+
+    const last = topics.at(-1)
+    const options =
+      topics.length === 0
+        ? []
+        : await selectRows<PhpbbPollOption>(
+            this.connection,
+            `select topic_id, poll_option_id, poll_option_text, poll_option_total
+               from \`${this.prefix}poll_options\`
+              where topic_id in (${topics.map(() => '?').join(', ')})
+              order by topic_id asc, poll_option_id asc`,
+            topics.map((topic) => topic.topic_id),
+          )
+
+    return {
+      rows: topics
+        .map((topic) => mapPhpbbPoll(topic, options))
+        .filter((poll): poll is NonNullable<typeof poll> => poll !== null),
+      nextCursor: topics.length < limit || last === undefined ? null : last.topic_id,
+    }
+  }
+
+  async pollVotes(afterId: number, limit: number): Promise<Page<ImportedPollVote>> {
+    const table = `\`${this.prefix}poll_votes\``
+    const page = await pageByGroup<PhpbbPollVote>({
+      afterGroup: afterId,
+      limit,
+      groupOf: (row) => row.topic_id,
+      fetch: (afterGroup, take) =>
+        selectRows(
+          this.connection,
+          `select topic_id, poll_option_id, vote_user_id from ${table}
+            where topic_id > ? order by topic_id asc, vote_user_id asc limit ?`,
+          [afterGroup, take],
+        ),
+      fetchGroup: (group) =>
+        selectRows(
+          this.connection,
+          `select topic_id, poll_option_id, vote_user_id from ${table}
+            where topic_id = ? order by vote_user_id asc`,
+          [group],
+        ),
+    })
+    return mapPage(page, mapPhpbbPollVote)
+  }
+
+  async privateMessages(afterId: number, limit: number): Promise<Page<ImportedPrivateMessage>> {
+    const messages = await selectRows<PhpbbPrivateMessage>(
+      this.connection,
+      `select msg_id, author_id, message_subject, message_text, bbcode_uid, message_time
+         from \`${this.prefix}privmsgs\`
+        where msg_id > ? order by msg_id asc limit ?`,
+      [afterId, limit],
+    )
+
+    const last = messages.at(-1)
+    const copies =
+      messages.length === 0
+        ? []
+        : await selectRows<PhpbbPrivateMessageCopy>(
+            this.connection,
+            `select msg_id, user_id, author_id, folder_id, pm_deleted, pm_unread
+               from \`${this.prefix}privmsgs_to\`
+              where msg_id in (${messages.map(() => '?').join(', ')})
+              order by msg_id asc, user_id asc`,
+            messages.map((message) => message.msg_id),
+          )
+
+    return {
+      rows: messages
+        .map((message) => mapPhpbbPrivateMessage(message, copies))
+        .filter((message): message is NonNullable<typeof message> => message !== null),
+      nextCursor: messages.length < limit || last === undefined ? null : last.msg_id,
+    }
+  }
+
+  threadSubscriptions(afterId: number, limit: number): Promise<Page<ImportedSubscription>> {
+    return this.#watch(`\`${this.prefix}topics_watch\``, 'topic_id', afterId, limit)
+  }
+
+  forumSubscriptions(afterId: number, limit: number): Promise<Page<ImportedSubscription>> {
+    return this.#watch(`\`${this.prefix}forums_watch\``, 'forum_id', afterId, limit)
+  }
+
+  reputation(): Promise<Page<ImportedReputation>> {
+    return noPhpbbReputation()
+  }
+
+  async warnings(afterId: number, limit: number): Promise<Page<ImportedWarning>> {
+    return mapPage(
+      await keysetPage<PhpbbWarning>(
+        this.connection,
+        `select warning_id, user_id, post_id, warning_time
+           from \`${this.prefix}warnings\``,
+        'warning_id',
+        (row) => row.warning_id,
+        afterId,
+        limit,
+      ),
+      mapPhpbbWarning,
+    )
+  }
+
+  async bans(afterId: number, limit: number): Promise<Page<ImportedBan>> {
+    return mapPage(
+      await keysetPage<PhpbbBan>(
+        this.connection,
+        `select ban_id, ban_userid, ban_start, ban_end, ban_exclude,
+                ban_reason, ban_give_reason
+           from \`${this.prefix}banlist\``,
+        'ban_id',
+        (row) => row.ban_id,
+        afterId,
+        limit,
+      ),
+      mapPhpbbBan,
+    )
+  }
+
+  async userRelations(afterId: number, limit: number): Promise<Page<ImportedUserRelation>> {
+    const table = `\`${this.prefix}zebra\``
+    const page = await pageByGroup<PhpbbZebra>({
+      afterGroup: afterId,
+      limit,
+      groupOf: (row) => row.user_id,
+      fetch: (afterGroup, take) =>
+        selectRows(
+          this.connection,
+          `select user_id, zebra_id, friend, foe from ${table}
+            where user_id > ? order by user_id asc, zebra_id asc limit ?`,
+          [afterGroup, take],
+        ),
+      fetchGroup: (group) =>
+        selectRows(
+          this.connection,
+          `select user_id, zebra_id, friend, foe from ${table}
+            where user_id = ? order by zebra_id asc`,
+          [group],
+        ),
+    })
+    return mapPage(page, mapPhpbbZebra)
+  }
+
+  async #watch(
+    table: string,
+    targetColumn: 'topic_id' | 'forum_id',
+    afterId: number,
+    limit: number,
+  ): Promise<Page<ImportedSubscription>> {
+    const page = await pageByGroup<PhpbbWatch>({
+      afterGroup: afterId,
+      limit,
+      groupOf: (row) => row.user_id,
+      fetch: (afterGroup, take) =>
+        selectRows(
+          this.connection,
+          `select user_id, ${targetColumn} as target_id from ${table}
+            where user_id > ? order by user_id asc, ${targetColumn} asc limit ?`,
+          [afterGroup, take],
+        ),
+      fetchGroup: (group) =>
+        selectRows(
+          this.connection,
+          `select user_id, ${targetColumn} as target_id from ${table}
+            where user_id = ? order by ${targetColumn} asc`,
+          [group],
+        ),
+    })
+    return mapPage(page, mapPhpbbWatch)
+  }
+}