@cloud411716/fancy-webnovel 2.2.0 → 3.0.3

package/docs/material-library/import.js

@@ -0,0 +1,437 @@
+#!/usr/bin/env node
+/**
+ * 网文创作素材库 · 原子素材 JSONL 校验 / 转 SurrealQL / 追加导入
+ *
+ * 用法：
+ *   node import.js --check sample.jsonl
+ *   node import.js --to-surql sample.surql sample.jsonl
+ *   node import.js --endpoint http://127.0.0.1:8000 --namespace fancy \
+ *     --database material_library --user root --pass root sample.jsonl
+ *
+ * 一条素材有 7 个字段：
+ *   id, kind, name, tags, genres, note, source
+ *
+ * 默认只追加：已存在的 id 或同 kind+name 会跳过；--update 才覆盖。
+ */
+
+import { readFileSync, writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { MATERIAL_KINDS } from '../../plugins/fancy-material/kinds.js';
+
+const HERE = fileURLToPath(new URL('.', import.meta.url));
+const ALLOWED_FIELDS = new Set(['id', 'kind', 'name', 'tags', 'genres', 'note', 'source']);
+const ID_RE = /^[a-z][a-z0-9_]*$/;
+
+class ImportError extends Error {
+  constructor(message, line) {
+    super(line ? `第 ${line} 行：${message}` : message);
+    this.line = line || null;
+  }
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function ensureString(value, field, line) {
+  if (typeof value !== 'string' || !value.trim()) {
+    throw new ImportError(`字段 ${field} 必须是非空字符串`, line);
+  }
+}
+
+function ensureStringArray(value, field, line, min, max) {
+  if (!Array.isArray(value)) throw new ImportError(`字段 ${field} 必须是数组`, line);
+  if (value.length < min || value.length > max) {
+    throw new ImportError(`字段 ${field} 需要 ${min}-${max} 项，实际 ${value.length} 项`, line);
+  }
+  for (const item of value) {
+    if (typeof item !== 'string' || !item.trim()) {
+      throw new ImportError(`字段 ${field} 的数组元素必须是非空字符串`, line);
+    }
+  }
+}
+
+function validateRecord(raw, line, seenIds) {
+  if (!isPlainObject(raw)) throw new ImportError('每行必须是一个 JSON 对象', line);
+
+  for (const key of Object.keys(raw)) {
+    if (!ALLOWED_FIELDS.has(key)) {
+      throw new ImportError(`不允许的字段：${key}。原子素材只允许 id/kind/name/tags/genres/note/source`, line);
+    }
+  }
+
+  ensureString(raw.id, 'id', line);
+  ensureString(raw.kind, 'kind', line);
+  ensureString(raw.name, 'name', line);
+  ensureStringArray(raw.tags, 'tags', line, 3, 8);
+  ensureStringArray(raw.genres, 'genres', line, 1, 3);
+  if (raw.note !== undefined) {
+    if (typeof raw.note !== 'string') throw new ImportError('note 必须是字符串', line);
+  }
+  const source = raw.source === undefined ? 'builtin' : raw.source;
+  if (source !== 'builtin' && source !== 'other') {
+    throw new ImportError("source 只能是 'builtin' 或 'other'", line);
+  }
+
+  const id = raw.id;
+  if (!ID_RE.test(id) || id.length > 120) {
+    throw new ImportError(`id 只能用小写字母、数字、下划线，且不超过 120 字符：${id}`, line);
+  }
+  const kind = raw.kind;
+  if (source === 'builtin') {
+    if (!MATERIAL_KINDS.includes(kind)) {
+      throw new ImportError(`builtin 素材的 kind 不在允许列表：${kind}`, line);
+    }
+    if (!id.startsWith(kind + '_')) {
+      throw new ImportError(`builtin 素材的 id 必须以 kind 开头：${kind}_...，实际 ${id}`, line);
+    }
+  }
+  if (seenIds.has(id)) throw new ImportError(`id 重复：${id}`, line);
+  seenIds.add(id);
+
+  return {
+    id,
+    kind,
+    record: {
+      source,
+      kind,
+      name: raw.name.trim(),
+      tags: raw.tags.map((item) => item.trim()),
+      genres: raw.genres.map((item) => item.trim()),
+      note: typeof raw.note === 'string' ? raw.note : '',
+    },
+  };
+}
+
+function parseArgs(argv) {
+  const env = process.env;
+  const opts = {
+    endpoint: env.SURREAL_ENDPOINT || env.SURREAL_URL || 'http://127.0.0.1:8000',
+    namespace: env.SURREAL_NAMESPACE || 'fancy',
+    database: env.SURREAL_DATABASE || 'material_library',
+    user: env.SURREAL_USER || 'root',
+    pass: env.SURREAL_PASS || '',
+    authLevel: env.SURREAL_AUTH_LEVEL || 'root',
+    check: false,
+    dryRun: false,
+    toSurql: null,
+    withSchema: false,
+    update: false,
+    prefetch: true,
+    quiet: false,
+    help: false,
+  };
+  const files = [];
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    const take = (name) => {
+      if (arg.includes('=')) return arg.slice(arg.indexOf('=') + 1);
+      const next = argv[i + 1];
+      if (next === undefined || (next.startsWith('-') && next !== '-')) throw new ImportError(`${name} 需要一个值`);
+      i += 1;
+      return next;
+    };
+    if (arg === '--help' || arg === '-h') opts.help = true;
+    else if (arg === '--check') opts.check = true;
+    else if (arg === '--dry-run') { opts.check = true; opts.dryRun = true; }
+    else if (arg.startsWith('--to-surql')) opts.toSurql = take('--to-surql');
+    else if (arg === '--with-schema') opts.withSchema = true;
+    else if (arg === '--update') opts.update = true;
+    else if (arg === '--no-prefetch') opts.prefetch = false;
+    else if (arg === '--quiet') opts.quiet = true;
+    else if (arg.startsWith('--endpoint')) opts.endpoint = take('--endpoint');
+    else if (arg.startsWith('--namespace') || arg.startsWith('--ns')) opts.namespace = take('--namespace');
+    else if (arg.startsWith('--database') || arg.startsWith('--db')) opts.database = take('--database');
+    else if (arg.startsWith('--user') || arg.startsWith('--username')) opts.user = take('--user');
+    else if (arg.startsWith('--pass') || arg.startsWith('--password')) opts.pass = take('--pass');
+    else if (arg.startsWith('--auth-level')) opts.authLevel = take('--auth-level');
+    else if (arg === '--') {
+      files.push(...argv.slice(i + 1));
+      break;
+    } else if (arg.startsWith('-')) {
+      throw new ImportError(`未知参数：${arg}`);
+    } else {
+      files.push(arg);
+    }
+  }
+  if (!files.length) files.push('sample.jsonl');
+  return { opts, files };
+}
+
+function loadJsonl(file, seenIds = new Set()) {
+  const text = readFileSync(file, 'utf8').replace(/^\uFEFF/, '');
+  const lines = text.split(/\r?\n/);
+  const records = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    let raw;
+    try {
+      raw = JSON.parse(line);
+    } catch (err) {
+      throw new ImportError(`JSON 解析失败：${err.message}`, i + 1);
+    }
+    records.push(validateRecord(raw, i + 1, seenIds));
+  }
+  if (!records.length) throw new ImportError(`${file} 里没有素材记录`);
+  return records;
+}
+
+function surrealLiteral(value) {
+  if (value === null) return 'NULL';
+  if (typeof value === 'string') return JSON.stringify(value);
+  if (typeof value === 'boolean') return value ? 'true' : 'false';
+  if (typeof value === 'number') {
+    if (!Number.isFinite(value)) throw new ImportError('JSON 里不能有 NaN 或 Infinity');
+    return JSON.stringify(value);
+  }
+  if (Array.isArray(value)) return '[' + value.map(surrealLiteral).join(', ') + ']';
+  if (isPlainObject(value)) {
+    return '{ ' + Object.entries(value)
+      .map(([key, item]) => `${JSON.stringify(key)}: ${surrealLiteral(item)}`)
+      .join(', ') + ' }';
+  }
+  throw new ImportError(`不支持的 JSON 值类型：${typeof value}`);
+}
+
+function toSurql(records, opts) {
+  const out = [];
+  if (opts.withSchema) {
+    out.push(readFileSync(resolve(HERE, 'schema.surql'), 'utf8').trimEnd());
+    out.push('');
+  }
+  for (const item of records) {
+    const literal = surrealLiteral(item.record);
+    out.push(`-- ${item.kind}: ${item.record.name}`);
+    if (opts.update) {
+      out.push(`UPSERT material:${item.id} MERGE ${literal};`);
+    } else {
+      const kind = surrealLiteral(item.kind);
+      const name = surrealLiteral(item.record.name);
+      out.push(`IF (SELECT VALUE id FROM material WHERE id = material:${item.id} OR (kind = ${kind} AND name = ${name})) = [] { CREATE material:${item.id} CONTENT ${literal}; };`);
+    }
+  }
+  return out.join('\n') + '\n';
+}
+
+function rpcUrl(endpoint) {
+  const base = String(endpoint || '').replace(/\/+$/, '');
+  return /\/rpc$/.test(base) ? base : base + '/rpc';
+}
+
+async function connectDb(opts) {
+  let Surreal;
+  let createRemoteEngines;
+  let RecordId;
+  try {
+    ({ Surreal, createRemoteEngines, RecordId } = await import('surrealdb'));
+  } catch {
+    throw new ImportError('缺少 surrealdb 依赖。请在 fancy-webnovel 仓库内运行，或先 npm install surrealdb。');
+  }
+  const db = new Surreal({ engines: { ...createRemoteEngines() } });
+  await db.connect(rpcUrl(opts.endpoint));
+  const signin = { username: opts.user, password: opts.pass };
+  if (opts.authLevel === 'namespace' || opts.authLevel === 'ns') signin.namespace = opts.namespace;
+  if (opts.authLevel === 'database' || opts.authLevel === 'db') {
+    signin.namespace = opts.namespace;
+    signin.database = opts.database;
+  }
+  await db.signin(signin);
+  await db.use({ namespace: opts.namespace, database: opts.database });
+  return { db, RecordId };
+}
+
+async function ensureSchema(db) {
+  const info = await db.query('INFO FOR TABLE material;');
+  const table = Array.isArray(info) && info[0] ? info[0] : {};
+  if (!table.fields || !table.fields.kind || !table.fields.tags) {
+    throw new ImportError('目标库还没有原子素材表结构，请先导入 schema.surql。');
+  }
+}
+
+async function existingMaterialState(db) {
+  const rows = await db.query('SELECT id, kind, name FROM material;');
+  const list = Array.isArray(rows) && Array.isArray(rows[0]) ? rows[0] : [];
+  const ids = new Set();
+  const names = new Set();
+  for (const row of list) {
+    const id = String(row.id || '').replace(/^material:/, '');
+    if (id) ids.add(id);
+    if (row.kind && row.name) names.add(String(row.kind) + '\u0000' + String(row.name));
+  }
+  return { ids, names };
+}
+
+function duplicateError(err) {
+  const message = err && err.message ? err.message : String(err);
+  return /already exists|already contains|duplicate|unique|index .*contains/i.test(message);
+}
+
+async function importToDb(records, opts) {
+  const { db, RecordId } = await connectDb(opts);
+  try {
+    await ensureSchema(db);
+    // Full-table prefetch is convenient for one-off imports, but a queue run
+    // imports thousands of small batches against a growing table. In that mode
+    // the unique indexes are the source of truth; duplicate CREATEs are caught
+    // below and counted as skipped.
+    const state = opts.prefetch
+      ? await existingMaterialState(db)
+      : { ids: new Set(), names: new Set() };
+    let imported = 0;
+    let skipped = 0;
+
+    for (const item of records) {
+      const pair = String(item.kind) + '\u0000' + String(item.record.name);
+      const exists = opts.prefetch && (state.ids.has(item.id) || state.names.has(pair));
+      if (exists && !opts.update) {
+        skipped += 1;
+        continue;
+      }
+      if (opts.dryRun) {
+        imported += 1;
+        state.ids.add(item.id);
+        state.names.add(pair);
+        continue;
+      }
+      if (opts.update) {
+        await db.upsert(new RecordId('material', item.id)).merge(item.record);
+      } else {
+        try {
+          await db.create(new RecordId('material', item.id)).content(item.record);
+        } catch (err) {
+          if (duplicateError(err)) {
+            skipped += 1;
+            continue;
+          }
+          throw err;
+        }
+      }
+      imported += 1;
+      state.ids.add(item.id);
+      state.names.add(pair);
+      if (!opts.quiet && imported % 100 === 0) {
+        process.stderr.write(`已导入 ${imported}/${records.length}\n`);
+      }
+    }
+    return { imported, skipped };
+  } finally {
+    try { await db.close(); } catch { /* ignore */ }
+  }
+}
+
+function usage() {
+  return `网文创作素材库 · 原子素材导入工具
+
+用法：
+  node import.js --check <file.jsonl>
+  node import.js --to-surql <out.surql> <file.jsonl>
+  node import.js [连接参数] <file.jsonl>
+
+一条素材字段：
+  id, kind, name, tags, genres, note, source
+
+source：
+  builtin  内置素材，kind 必须是允许列表之一
+  other    自定义素材，kind 不受限制
+
+连接参数：
+  --endpoint <url>      默认 $SURREAL_ENDPOINT 或 http://127.0.0.1:8000
+  --namespace <ns>      默认 $SURREAL_NAMESPACE 或 fancy
+  --database <db>       默认 $SURREAL_DATABASE 或 material_library
+  --user <user>         默认 $SURREAL_USER 或 root
+  --pass <pass>         默认 $SURREAL_PASS
+  --auth-level <level>  root | namespace | database（默认 root）
+
+其他：
+  --check               只校验 JSONL，不连接数据库
+  --dry-run             连接数据库、检查 schema，但不写入
+  --to-surql <file>     转成 SurrealQL；file 为 - 时输出到 stdout
+  --with-schema         与 --to-surql 连用时，把 schema.surql 一起写进去
+  --update              显式覆盖已存在的 id（默认只追加，不覆盖）
+  --no-prefetch         不预读全表，依赖唯一索引去重（批量队列用）
+  --quiet               不打印进度
+  -h, --help            显示本帮助
+
+示例：
+  node docs/material-library/import.js --check docs/material-library/sample.jsonl
+  node docs/material-library/import.js --to-surql /tmp/materials.surql docs/material-library/sample.jsonl
+  node docs/material-library/import.js --endpoint http://127.0.0.1:8000 --user root --pass root docs/material-library/sample.jsonl
+`;
+}
+
+async function main() {
+  let parsed;
+  try {
+    parsed = parseArgs(process.argv.slice(2));
+  } catch (err) {
+    process.stderr.write(String(err.message) + '\n\n' + usage());
+    process.exitCode = 2;
+    return;
+  }
+  const { opts, files } = parsed;
+  if (opts.help) {
+    process.stdout.write(usage());
+    return;
+  }
+
+  let records;
+  try {
+    const seenIds = new Set();
+    records = files.flatMap((file) => loadJsonl(file, seenIds));
+  } catch (err) {
+    process.stderr.write('❌ ' + err.message + '\n');
+    process.exitCode = 1;
+    return;
+  }
+
+  if (opts.toSurql !== null) {
+    let output;
+    try {
+      output = toSurql(records, opts);
+    } catch (err) {
+      process.stderr.write('❌ ' + err.message + '\n');
+      process.exitCode = 1;
+      return;
+    }
+    if (opts.toSurql === '-' || opts.toSurql === '') {
+      process.stdout.write(output);
+    } else {
+      writeFileSync(opts.toSurql, output, 'utf8');
+      if (!opts.quiet) process.stderr.write(`✅ 已写入 ${opts.toSurql}（${records.length} 条素材）\n`);
+    }
+    return;
+  }
+
+  if (opts.check && !opts.dryRun) {
+    if (!opts.quiet) {
+      const kinds = new Map();
+      for (const item of records) kinds.set(item.kind, (kinds.get(item.kind) || 0) + 1);
+      process.stdout.write(`✅ 校验通过：${records.length} 条素材，${kinds.size} 个 kind\n`);
+      for (const [kind, count] of [...kinds.entries()].sort()) {
+        process.stdout.write(`  ${kind}: ${count}\n`);
+      }
+    }
+    return;
+  }
+
+  let result;
+  try {
+    result = await importToDb(records, opts);
+  } catch (err) {
+    process.stderr.write('❌ 导入失败：' + err.message + '\n');
+    process.exitCode = 1;
+    return;
+  }
+  if (opts.dryRun) {
+    process.stdout.write(`✅ 预检通过：预计新增 ${result.imported} 条，跳过 ${result.skipped} 条已存在素材；schema 可用，未写入。\n`);
+  } else {
+    process.stdout.write(`✅ 导入完成：新增 ${result.imported} 条素材，跳过 ${result.skipped} 条已存在素材。\n`);
+  }
+}
+
+main().catch((err) => {
+  process.stderr.write('❌ ' + (err && err.stack ? err.stack : String(err)) + '\n');
+  process.exitCode = 1;
+});