@firok-arc-project/arc-centrifuge 0.5.0

package/readme.md

@@ -0,0 +1,74 @@
+# Centrifuge
+
+用于识别和处理指定目录内的博客文档文件.
+
+设计初衷是制作一个脱离特定客户端环境且 markdown-first 的博文处理工具.
+
+使用时需要指定一个 JSON5 数据文件, 这个文件包含如下字段:
+
+```json5
+{
+  "source_folder": "./input", // 一个相对路径, 指定从哪里读取文档数据
+  "target_folder": "./output", // 一个相对路径, 指定将各个索引数据写入到哪里
+  "tags": [
+    {
+      // 标签数据
+    },
+    // ...
+  ], // 标签列表
+}
+```
+
+根据文档数据, 将生成如下结构:
+
+```text
+{target_folder}/
+├─ all-tag.json  所有的标签数据
+│
+├─ tag-indexing-{tag_id}-all.json  指定标签的索引全部数据
+├─ tag-indexing-{tag_id}-0.json  指定标签的索引分页数据, 第 1 页
+├─ tag-indexing-{tag_id}-1.json  指定标签的索引分页数据, 第 2 页
+├─ ...
+│
+├─ timeline-indexing-all.json  按照时间线排序的索引全部数据
+├─ timeline-indexing-0.json  按照时间线排序的索引分页数据, 第 1 页
+├─ timeline-indexing-1.json  按照时间线排序的索引分页数据, 第 2 页
+└─ ...
+```
+
+生成的内容不会包含各个文档的原始数据.
+
+## 文档配置
+
+此工具会递归加载指定目录树内所有文件, 按照不同格式将对其进行不同处理.
+
+### 文档 front-matter 数据
+
+每个 `.md` 文件对应一篇可渲染的博文.
+
+我们使用 `gray-matter` 读取和处理其中的 front-matter,
+将所有的数据整理成若干个 JSON 数据文件输出到指定目录.
+
+要正常处理一个文档文件, 它的 front-matter 中需要包含下述字段:
+
+* `id: string` 文档唯一 ID. 这会用于生成唯一 URL
+* `title: string` 文档标题
+* `createTimestamp: string` 文档创建时间
+* `updateTimestamp: string` 文档更新时间
+* `sortTimestamp: string` 文档索引排序时间
+* `tags: string[]` 文档标签. 值需要是正确配置了的标签 ID 列表
+
+原始的 front-matter 数据会作为文档的简述数据输出到各个索引数据中.
+
+### 标签数据
+
+标签数据包含如下字段:
+
+* `id: string` 标签唯一 ID
+* `name: string` 标签名称
+* `desc: string` 标签描述
+* `hidden: boolean` 是否是隐藏标签
+* `indexing: boolean` 是否为包含此标签的博文创建索引列表
+* `indexingPagination: boolean` 为此标签创建索引列表时是否需要分页
+* `indexingPaginationSize: int` 为此标签创建索引列表时每页的博文数量
+

package/src/doc.ts

@@ -0,0 +1,92 @@
+
+import * as fs from 'node:fs'
+
+import matter from 'gray-matter'
+import {FolderNode} from './types/file-system-def'
+import {DocMeta, DocMetaMap} from './types/doc-meta-def'
+
+/**
+ * 读取文档中的元数据
+ * */
+function loadDocMeta(
+  textDoc: string,
+  pathRelative: string,
+): DocMeta
+{
+  const fileMatter = matter(textDoc)
+  // console.log('matter', fileMatter)
+  const json = fileMatter.data as DocMeta
+  let {
+    id,
+    title,
+    createTimestamp,
+    updateTimestamp,
+    sortTimestamp,
+    tags,
+  } = json
+
+  if('string' !== typeof id)
+    throw 'doc id not found: ' + json
+  if('string' !== typeof title)
+    throw 'doc title not found: ' + json
+
+  const dateCreate = new Date(json.createTimestamp)
+  const dateUpdate = new Date(json.updateTimestamp)
+  const dateSort = new Date(json.sortTimestamp)
+  createTimestamp = dateCreate.getTime()
+  updateTimestamp = dateUpdate.getTime()
+  sortTimestamp = dateSort.getTime()
+
+  if(!Array.isArray(tags))
+    tags = []
+
+  return {
+    ...json,
+    id,
+    title,
+    createTimestamp,
+    updateTimestamp,
+    sortTimestamp,
+    tags,
+    pathRelative,
+  } as DocMeta
+}
+
+function findDocData(folder: FolderNode, map: DocMetaMap): void
+{
+  for(const file of folder.listChildrenFile)
+  {
+    if(file.filename.endsWith('.md'))
+    {
+      const textFile = fs.readFileSync(file.pathAbsolute, {
+        encoding: 'utf-8',
+      })
+
+      const dataDoc = loadDocMeta(textFile, file.pathRelative)
+
+      if(map[dataDoc.id] != null)
+        throw 'doc id duplicated: ' + dataDoc.id
+
+      map[dataDoc.id] = dataDoc
+    }
+  }
+
+  for(const childFolder of folder.listChildrenFolder)
+  {
+    findDocData(childFolder, map)
+  }
+}
+
+export async function collectDocMeta(folder: FolderNode): Promise<DocMetaMap>
+{
+  const ret: DocMetaMap = {}
+  findDocData(folder, ret)
+  return ret
+}
+
+export function getSortedDocMeta(map: DocMetaMap): DocMeta[]
+{
+  return Object.values(map).sort((a, b) => {
+    return a.sortTimestamp - b.sortTimestamp
+  })
+}

package/src/file-system.ts

@@ -0,0 +1,59 @@
+
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+import {Dirent} from 'node:fs'
+import {FileNode, FolderNode} from './types/file-system-def'
+
+export function readFileSystem(
+  pathDir: string,
+  pathBaseDir?: string,
+): FolderNode
+{
+  if(pathBaseDir == null)
+    pathBaseDir = pathDir
+
+  const listDirent: Dirent[] = fs.readdirSync(pathDir, {
+    withFileTypes: true,
+  })
+  const pathAbsolute = path.resolve(pathDir)
+  const pathRelative = path.relative(pathBaseDir, pathDir)
+  console.log(
+    'pathBaseDir', pathBaseDir,
+      'pathDir', pathDir,
+      'pathAbsolute', pathAbsolute,
+      'pathRelative', pathRelative,
+    )
+
+  const listFolder: FolderNode[] = []
+  const listFile: FileNode[] = []
+
+  for(const obj of listDirent)
+  {
+    const pathAbsolute = path.resolve(pathDir, obj.name)
+    const pathRelative = path.relative(pathBaseDir, pathAbsolute)
+
+    if(obj.isFile())
+    {
+      const filename = obj.name
+      const fileExt = filename.split('.').pop()
+      listFile.push({
+        filename,
+        fileExt,
+        pathAbsolute,
+        pathRelative,
+      } as FileNode)
+    }
+    else if(obj.isDirectory())
+    {
+      const folderNode = readFileSystem(pathAbsolute, pathBaseDir)
+      listFolder.push(folderNode)
+    }
+  }
+
+  return {
+    listChildrenFile: listFile,
+    listChildrenFolder: listFolder,
+    pathAbsolute,
+    pathRelative,
+  } as FolderNode
+}

package/src/gen-all-tag-json.ts

@@ -0,0 +1,11 @@
+import {TagMap} from './types/tag-data-def'
+import {write, resolve} from './gen.js'
+
+export async function genAllTagJson(mapTag: TagMap, pathFolderTarget: string): Promise<void>
+{
+  const pathFileTarget = resolve(pathFolderTarget, 'all-tag.json')
+
+  let content: string = JSON.stringify(Object.values(mapTag))
+
+  await write(pathFileTarget, content)
+}

package/src/gen-tag-indexing.ts

@@ -0,0 +1,50 @@
+import {TagData, TagMap} from './types/tag-data-def'
+import {DocMeta, DocMetaMap} from './types/doc-meta-def'
+import {resolve, write} from './gen.js'
+
+/**
+ * 为某个标签生成索引内容
+ * */
+async function genTagIndexingSingle(
+  tag: TagData,
+  listSortedDocMeta: DocMeta[],
+  pathFolderTarget: string,
+)
+{
+  if(!tag.indexing)
+    return
+
+  const tagId = tag.id
+
+  const listDocMeta =
+    listSortedDocMeta.filter(docMeta => docMeta.tags.includes(tagId))
+
+  const pathFileIndexingAll = resolve(pathFolderTarget, `tag-indexing-${tagId}-all.json`)
+  await write(pathFileIndexingAll, listDocMeta)
+
+  if(!tag.indexingPagination)
+    return
+
+  const pageSize = tag.indexingPaginationSize
+  const pageCount = Math.ceil(listDocMeta.length / pageSize)
+  for(let pageIndex = 0; pageIndex < pageCount; pageIndex++)
+  {
+    const start = pageIndex * pageSize
+    const end = Math.min(start + pageSize, listDocMeta.length)
+    const listDocMetaPage = listDocMeta.slice(start, end)
+    const pathFileIndexingPage = resolve(pathFolderTarget, `tag-indexing-${tagId}-${pageIndex}.json`)
+    await write(pathFileIndexingPage, listDocMetaPage)
+  }
+}
+
+export async function genTagIndexing(
+  mapTag: TagMap,
+  listSortedDocMeta: DocMeta[],
+  pathFolderTarget: string
+): Promise<void>
+{
+  for(const tag of Object.values(mapTag))
+  {
+    await genTagIndexingSingle(tag, listSortedDocMeta, pathFolderTarget)
+  }
+}

package/src/gen-timeline-indexing.ts

@@ -0,0 +1,27 @@
+import {DocMeta} from './types/doc-meta-def'
+import {resolve, write} from './gen.js'
+import { MetaConfig } from './types/meta-config-def'
+
+export async function genTimelineIndexing(
+  listSortedDocMeta: DocMeta[],
+  pathFolderTarget: string,
+  metaConfig: MetaConfig,
+): Promise<void>
+{
+  const pathFileIndexingAll = resolve(pathFolderTarget, 'timeline-indexing-all.json')
+  await write(pathFileIndexingAll, listSortedDocMeta)
+
+  if(!metaConfig.timelinePagination)
+    return
+
+  const pageSize = metaConfig.timelinePaginationSize
+  const pageCount = Math.ceil(listSortedDocMeta.length / pageSize)
+  for(let pageIndex = 0; pageIndex < pageCount; pageIndex++)
+  {
+    const start = pageIndex * pageSize
+    const end = Math.min(start + pageSize, listSortedDocMeta.length)
+    const listDocMeta = listSortedDocMeta.slice(start, end)
+    const pathFileIndexing = resolve(pathFolderTarget, `timeline-indexing-${pageIndex}.json`)
+    await write(pathFileIndexing, listDocMeta)
+  }
+}

package/src/gen.ts

@@ -0,0 +1,26 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+export async function write(pathFile: string, content: string | object)
+{
+  const pathFolderParent = path.dirname(pathFile)
+  if(!fs.existsSync(pathFolderParent))
+    fs.mkdirSync(pathFolderParent, { recursive: true })
+
+  content = 'object' === typeof content ? JSON.stringify(content) : content
+  fs.writeFileSync(
+    pathFile,
+    content,
+    { encoding: 'utf8', },
+  )
+}
+
+export function resolve(pathFrom: string, pathTo: string): string
+{
+  return path.resolve(pathFrom, pathTo)
+}
+
+export function relative(pathFrom: string, pathTo: string): string
+{
+  return path.relative(pathFrom, pathTo)
+}

package/src/id-entity.ts

@@ -0,0 +1,11 @@
+import {IdEntity, IdMap} from './types/id-entity-def'
+
+export function toMap<TypeEntity extends IdEntity>(list: TypeEntity[]): IdMap<TypeEntity>
+{
+  const ret: IdMap<TypeEntity> = {}
+  for(const entity of list)
+  {
+    ret[entity.id] = entity
+  }
+  return ret
+}

package/src/index.ts

@@ -0,0 +1,42 @@
+
+import * as process from 'node:process'
+import * as path from 'node:path'
+import {readFileSystem} from './file-system.js'
+import {readMetaConfig} from './meta-config.js'
+import {TagMap} from './types/tag-data-def'
+import {toMap} from './id-entity.js'
+import {DocMetaMap} from './types/doc-meta-def'
+import {collectDocMeta, getSortedDocMeta} from './doc.js'
+import {genAllTagJson} from './gen-all-tag-json.js'
+import {genTagIndexing} from './gen-tag-indexing.js'
+import {genTimelineIndexing} from './gen-timeline-indexing.js'
+
+export async function main()
+{
+  // 获取命令行参数
+  const args: string[] = process.argv.slice(2)
+
+  if(args.length < 1)
+    throw '未指定配置文件路径'
+
+  // 始终把第一个字符串当做本次识别和处理的基础目录
+  const pathMetaConfig = path.resolve(args[0])
+  const metaConfig = await readMetaConfig(pathMetaConfig)
+
+  const mapTag: TagMap = toMap(metaConfig.tags) // 处理 tag 数据
+
+  const folderSource = readFileSystem(metaConfig.sourceFolder)
+  const mapDocMeta: DocMetaMap = await collectDocMeta(folderSource)
+  const listSortedDocMeta = getSortedDocMeta(mapDocMeta)
+
+  console.log('mapTag', mapTag)
+  console.log('mapDocMeta', mapDocMeta)
+
+  // 开始生成并写入索引数据
+  const pathFolderTarget = metaConfig.targetFolder
+  await genAllTagJson(mapTag, pathFolderTarget)
+  await genTimelineIndexing(listSortedDocMeta, pathFolderTarget, metaConfig)
+  await genTagIndexing(mapTag, listSortedDocMeta, pathFolderTarget)
+}
+
+await main()

package/src/meta-config.ts

@@ -0,0 +1,76 @@
+import {MetaConfig} from './types/meta-config-def'
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+import JSON5 from 'json5'
+
+export async function readMetaConfig(pathConfig: string): Promise<MetaConfig>
+{
+  pathConfig = path.resolve(pathConfig)  // 把 pathConfig 处理成绝对路径
+
+  const textMetaConfig = fs.readFileSync(pathConfig, {
+    encoding: 'utf8',
+  })
+  const jsonMetaConfig = JSON5.parse(textMetaConfig) as MetaConfig
+  let {
+    sourceFolder,
+    targetFolder,
+    tags,
+    timelinePagination,
+    timelinePaginationSize,
+  } = jsonMetaConfig
+
+  // 把 sourceFolder 和 targetFolder 处理成绝对路径
+  sourceFolder = path.resolve(path.dirname(pathConfig), sourceFolder)
+  targetFolder = path.resolve(path.dirname(pathConfig), targetFolder)
+
+  if('string' !== typeof sourceFolder)
+    throw 'sourceFolder not found: ' + jsonMetaConfig
+  if('string' !== typeof targetFolder)
+    throw 'targetFolder not found: ' + jsonMetaConfig
+  if(!Array.isArray(tags))
+    tags = []
+
+  timelinePagination = timelinePagination ?? true
+  timelinePaginationSize = timelinePaginationSize ?? 10
+
+  const setTagId = new Set<string>()
+  for(const tag of tags)
+  {
+    let {
+      id,
+      name,
+      desc,
+      icon,
+      hidden,
+      indexing,
+      indexingPagination,
+      indexingPaginationSize,
+    } = tag
+
+    if('string' !== typeof id)
+      throw 'tag id not found: ' + tag
+    if(setTagId.has(id))
+      throw 'tag id duplicated: ' + id
+    setTagId.add(id)
+
+    if('string' !== typeof name)
+      tag.name = null
+    if('string' !== typeof desc)
+      tag.desc = null
+    if('string' !== typeof icon)
+      tag.icon = null
+
+    tag.hidden = !!hidden
+    tag.indexing = indexing ?? false
+    tag.indexingPagination = indexingPagination ?? true
+    tag.indexingPaginationSize = indexingPaginationSize ?? 10
+  }
+
+  return {
+    sourceFolder,
+    targetFolder,
+    tags,
+    timelinePagination,
+    timelinePaginationSize,
+  } as MetaConfig
+}

package/src/types/doc-meta-def.d.ts

@@ -0,0 +1,17 @@
+
+/**
+ * 文档的元数据
+ * */
+export interface DocMeta
+{
+  id: string
+  title: string
+  subtitle?: string
+  createTimestamp: number
+  updateTimestamp: number
+  sortTimestamp: number
+  tags: string[]
+  pathRelative: string
+}
+
+export type DocMetaMap = Record<string, DocMeta>

package/src/types/file-system-def.d.ts

@@ -0,0 +1,25 @@
+
+export declare interface FileSystemNode
+{
+  /**
+   * 完整路径
+   * */
+  pathAbsolute: string
+
+  /**
+   * 相对路径
+   * */
+  pathRelative: string
+}
+
+export declare interface FileNode extends FileSystemNode
+{
+  filename: string
+  fileExt: string
+}
+
+export declare interface FolderNode extends FileSystemNode
+{
+  listChildrenFolder: FolderNode[]
+  listChildrenFile: FileNode[]
+}

package/src/types/id-entity-def.d.ts

@@ -0,0 +1,7 @@
+
+export declare interface IdEntity
+{
+  id: string
+}
+
+export declare type IdMap<TypeEntity> = Record<string, TypeEntity>

package/src/types/meta-config-def.d.ts

@@ -0,0 +1,11 @@
+
+import {TagData} from './tag-data-def'
+
+export declare interface MetaConfig
+{
+  sourceFolder: string
+  targetFolder: string
+  tags: TagData[],
+  timelinePagination?: boolean
+  timelinePaginationSize?: number
+}

package/src/types/tag-data-def.d.ts

@@ -0,0 +1,44 @@
+import {IdEntity, IdMap} from './id-entity-def'
+
+/**
+ * 标签数据
+ * */
+export declare interface TagData extends IdEntity
+{
+  /**
+   * 标签名称
+   * */
+  name?: string
+
+  /**
+   * 标签图标
+   * */
+  icon?: string
+
+  /**
+   * 标签描述
+   * */
+  desc?: string
+
+  /**
+   * 是否是隐藏标签
+   * */
+  hidden?: boolean
+
+  /**
+   * 是否为包含此标签的博文创建索引列表
+   * */
+  indexing?: boolean
+
+  /**
+   * 为此标签创建索引列表时是否需要分页
+   * */
+  indexingPagination?: boolean
+
+  /**
+   * 为此标签创建索引列表时每页包含多少条信息
+   * */
+  indexingPaginationSize?: number
+}
+
+export declare type TagMap = IdMap<TagData>

package/src/types/tag-indexing-def.d.ts

@@ -0,0 +1,16 @@
+import {DocMeta} from './doc-meta-def'
+
+export declare interface TagIndexingPage
+{
+  pageIndex: number
+  listDocMeta: DocMeta[]
+}
+
+export declare interface TagIndexingData
+{
+  tagId: string
+  pageSize: number
+  pageCount: number
+  listPage: TagIndexingPage[]
+}
+

package/tsconfig.json

@@ -0,0 +1,61 @@
+{
+  // Visit https://aka.ms/tsconfig to read more about this file
+  "compilerOptions": {
+    // File Layout
+     "rootDir": "./src",
+     "outDir": "./dist",
+
+    // Environment Settings
+    // See also https://aka.ms/tsconfig/module
+    "module": "preserve",
+    "target": "es2020",
+    "types": [
+      "@types/node"
+    ],
+    // For nodejs:
+    // "lib": ["esnext"],
+    // "types": ["node"],
+    // and npm install -D @types/node
+
+    // Other Outputs
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+
+    // Stricter Typechecking Options
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": false,
+
+    // Style Options
+    "noImplicitReturns": false,
+    "noImplicitOverride": false,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noFallthroughCasesInSwitch": false,
+    "noPropertyAccessFromIndexSignature": false,
+
+    // Recommended Options
+    "strict": false,
+    "jsx": "react-jsx",
+    "verbatimModuleSyntax": false,
+    "isolatedModules": true,
+    "noUncheckedSideEffectImports": true,
+    "moduleDetection": "force",
+    "skipLibCheck": true,
+
+    // custom
+
+    "allowJs": true,
+    "allowSyntheticDefaultImports": true,
+    "allowUnreachableCode": true,
+    "allowUnusedLabels": true,
+    "checkJs": true,
+    "strictNullChecks": false,
+    "moduleResolution": "node",
+    "preserveConstEnums": true,
+    "preserveSymlinks": true
+  },
+  "include": [
+    "./src/*"
+  ],
+}