@ssobig/writer-cli 0.2.0

package/tools/writer-cli/src/local-index-storage.cjs

@@ -0,0 +1,284 @@
+"use strict";
+
+const fs = require("node:fs");
+const { canonicalJson } = require("./json.cjs");
+const { cliError } = require("./errors.cjs");
+const {
+  INDEX_SCHEMA_VERSION,
+  SERVER_SNAPSHOT_SOURCE,
+  trigramValues
+} = require("./local-index-snapshot.cjs");
+const { appendSqlIdentityFilters } = require("./local-index-query.cjs");
+const { resolveStoragePaths, createJsonBackend } = require("./local-index-cache.cjs");
+
+function metadataToRows(metadata) {
+  return Object.entries({
+    schema_version: metadata.schemaVersion,
+    source: metadata.source,
+    actor_json: JSON.stringify(metadata.actor),
+    synced_at: metadata.syncedAt,
+    last_verified_auth_at: metadata.lastVerifiedAuthAt || "",
+    snapshot_hash: metadata.snapshotHash,
+    project_version_count: String(metadata.projectVersionCount),
+    component_count: String(metadata.componentCount),
+    scalar_count: String(metadata.scalarCount)
+  });
+}
+
+function rowsToMetadata(rows) {
+  const values = Object.fromEntries(rows.map(row => [row.key, row.value]));
+  if (!values.schema_version) return null;
+  // 로컬 인덱스는 언제든 재구축 가능한 파생물이라 형식이 어긋나면 막지 않고 미동기화로 취급한다.
+  if (values.schema_version !== INDEX_SCHEMA_VERSION || values.source !== SERVER_SNAPSHOT_SOURCE) return null;
+  return {
+    schemaVersion: values.schema_version,
+    source: values.source,
+    actor: JSON.parse(values.actor_json || "null"),
+    syncedAt: values.synced_at,
+    lastVerifiedAuthAt: values.last_verified_auth_at || null,
+    snapshotHash: values.snapshot_hash,
+    projectVersionCount: Number(values.project_version_count || 0),
+    componentCount: Number(values.component_count || 0),
+    scalarCount: Number(values.scalar_count || 0)
+  };
+}
+
+// 현재 DDL이 기대하는 테이블 형태. 디스크의 실제 형태와 비교해 스키마 드리프트를 판정한다.
+const EXPECTED_TABLE_COLUMNS = Object.freeze({
+  metadata: Object.freeze(["key", "value"]),
+  components: Object.freeze([
+    "component_checksum", "component_set_revision", "data_hash", "data_json", "instance_id",
+    "instance_revision", "project_id", "project_slug", "template_id", "version_number"
+  ]),
+  scalar_nodes: Object.freeze([
+    "component_checksum", "component_set_revision", "data_hash", "field_name", "folded_text", "id",
+    "instance_id", "instance_revision", "match_id", "pointer", "project_id", "project_slug",
+    "search_text", "template_id", "value_json", "value_text", "value_type", "version_number"
+  ]),
+  scalar_trigrams: Object.freeze(["node_id", "trigram"])
+});
+
+function tableColumns(database, table) {
+  try {
+    return database.prepare(`PRAGMA table_info(${table})`).all().map(row => String(row.name)).sort();
+  } catch (error) {
+    void error;
+    return [];
+  }
+}
+
+function hasSchemaDrift(database) {
+  for (const [table, expected] of Object.entries(EXPECTED_TABLE_COLUMNS)) {
+    const actual = tableColumns(database, table);
+    if (actual.length === 0) continue;
+    if (actual.length !== expected.length || actual.some((name, position) => name !== expected[position])) return true;
+  }
+  if (tableColumns(database, "metadata").length === 0) return false;
+  try {
+    const values = Object.fromEntries(database
+      .prepare("SELECT key, value FROM metadata WHERE key IN ('schema_version', 'source')")
+      .all()
+      .map(row => [row.key, row.value]));
+    if (!values.schema_version) return false;
+    return values.schema_version !== INDEX_SCHEMA_VERSION || values.source !== SERVER_SNAPSHOT_SOURCE;
+  } catch (error) {
+    void error;
+    return true;
+  }
+}
+
+function dropIndexTables(database) {
+  try { database.exec("DROP TABLE IF EXISTS scalar_fts;"); } catch (ftsError) { void ftsError; }
+  database.exec(`
+    DROP TABLE IF EXISTS scalar_trigrams;
+    DROP TABLE IF EXISTS scalar_nodes;
+    DROP TABLE IF EXISTS components;
+    DROP TABLE IF EXISTS metadata;
+  `);
+}
+
+function createSqliteBackend(options) {
+  let sqlite;
+  try {
+    sqlite = require("node:sqlite");
+  } catch (error) {
+    throw cliError("E_LOCAL_INDEX_SQLITE_UNAVAILABLE", "이 Node.js 런타임에서 node:sqlite를 사용할 수 없습니다.", null, error);
+  }
+  const storage = resolveStoragePaths(options, "sqlite");
+  let database;
+  let ftsEnabled = false;
+  try {
+    database = new sqlite.DatabaseSync(storage.filePath);
+    fs.chmodSync(storage.filePath, 0o600);
+    database.exec("PRAGMA foreign_keys = ON; PRAGMA journal_mode = DELETE; PRAGMA synchronous = FULL; PRAGMA secure_delete = ON; PRAGMA temp_store = MEMORY;");
+    // 구버전 CLI가 만든 테이블은 CREATE TABLE IF NOT EXISTS로 갱신되지 않으므로 드리프트가 있으면 통째로 재생성한다.
+    if (hasSchemaDrift(database)) dropIndexTables(database);
+    database.exec(`
+      CREATE TABLE IF NOT EXISTS metadata (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      ) WITHOUT ROWID;
+      CREATE TABLE IF NOT EXISTS components (
+        project_id TEXT NOT NULL,
+        project_slug TEXT NOT NULL,
+        version_number INTEGER NOT NULL,
+        component_set_revision INTEGER NOT NULL,
+        component_checksum TEXT NOT NULL,
+        instance_id TEXT NOT NULL,
+        template_id TEXT NOT NULL,
+        instance_revision INTEGER NOT NULL,
+        data_hash TEXT NOT NULL,
+        data_json TEXT NOT NULL,
+        PRIMARY KEY (project_id, version_number, instance_id)
+      ) WITHOUT ROWID;
+      CREATE TABLE IF NOT EXISTS scalar_nodes (
+        id INTEGER PRIMARY KEY,
+        match_id TEXT NOT NULL UNIQUE,
+        project_id TEXT NOT NULL,
+        project_slug TEXT NOT NULL,
+        version_number INTEGER NOT NULL,
+        component_set_revision INTEGER NOT NULL,
+        component_checksum TEXT NOT NULL,
+        instance_id TEXT NOT NULL,
+        template_id TEXT NOT NULL,
+        instance_revision INTEGER NOT NULL,
+        data_hash TEXT NOT NULL,
+        pointer TEXT NOT NULL,
+        field_name TEXT NOT NULL,
+        value_type TEXT NOT NULL,
+        value_json TEXT NOT NULL,
+        value_text TEXT NOT NULL,
+        search_text TEXT NOT NULL,
+        folded_text TEXT NOT NULL,
+        FOREIGN KEY (project_id, version_number, instance_id)
+          REFERENCES components(project_id, version_number, instance_id) ON DELETE CASCADE
+      );
+      CREATE TABLE IF NOT EXISTS scalar_trigrams (
+        trigram TEXT NOT NULL,
+        node_id INTEGER NOT NULL REFERENCES scalar_nodes(id) ON DELETE CASCADE,
+        PRIMARY KEY (trigram, node_id)
+      ) WITHOUT ROWID;
+      CREATE INDEX IF NOT EXISTS scalar_nodes_project ON scalar_nodes(project_id, version_number, instance_id);
+      CREATE INDEX IF NOT EXISTS scalar_nodes_slug ON scalar_nodes(project_slug, version_number, instance_id);
+      CREATE INDEX IF NOT EXISTS scalar_nodes_template ON scalar_nodes(template_id);
+      CREATE INDEX IF NOT EXISTS scalar_nodes_pointer ON scalar_nodes(pointer);
+      CREATE INDEX IF NOT EXISTS scalar_nodes_field ON scalar_nodes(field_name);
+      CREATE INDEX IF NOT EXISTS scalar_nodes_exact ON scalar_nodes(value_type, value_json);
+    `);
+    try {
+      database.exec("CREATE VIRTUAL TABLE IF NOT EXISTS scalar_fts USING fts5(folded_text, tokenize='trigram');");
+      database.exec("BEGIN IMMEDIATE; DELETE FROM scalar_fts; INSERT INTO scalar_fts(rowid, folded_text) SELECT id, folded_text FROM scalar_nodes; COMMIT;");
+      ftsEnabled = true;
+    } catch (ftsError) {
+      try { database.exec("ROLLBACK"); } catch (rollbackError) { void rollbackError; }
+      ftsEnabled = false;
+    }
+  } catch (error) {
+    try { database?.close(); } catch (closeError) { void closeError; }
+    throw cliError("E_LOCAL_INDEX_SQLITE_UNAVAILABLE", "SQLite 로컬 인덱스를 열 수 없습니다.", null, error);
+  }
+
+  const statements = {
+    insertMetadata: database.prepare("INSERT INTO metadata(key, value) VALUES(?, ?)"),
+    insertComponent: database.prepare(`INSERT INTO components(
+      project_id, project_slug, version_number, component_set_revision, component_checksum,
+      instance_id, template_id, instance_revision, data_hash, data_json
+    ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
+    insertNode: database.prepare(`INSERT INTO scalar_nodes(
+      match_id, project_id, project_slug, version_number, component_set_revision, component_checksum,
+      instance_id, template_id, instance_revision, data_hash,
+      pointer, field_name, value_type, value_json, value_text, search_text, folded_text
+    ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`),
+    insertTrigram: database.prepare("INSERT OR IGNORE INTO scalar_trigrams(trigram, node_id) VALUES(?, ?)"),
+    insertFts: ftsEnabled ? database.prepare("INSERT INTO scalar_fts(rowid, folded_text) VALUES(?, ?)") : null,
+    metadata: database.prepare("SELECT key, value FROM metadata ORDER BY key"),
+    counts: database.prepare("SELECT (SELECT COUNT(*) FROM components) AS components, (SELECT COUNT(*) FROM scalar_nodes) AS scalars")
+  };
+
+  function replaceState(state) {
+    database.exec("BEGIN IMMEDIATE");
+    try {
+      if (ftsEnabled) database.exec("DELETE FROM scalar_fts;");
+      database.exec("DELETE FROM scalar_trigrams; DELETE FROM scalar_nodes; DELETE FROM components; DELETE FROM metadata;");
+      for (const [key, value] of metadataToRows(state.metadata)) statements.insertMetadata.run(key, value);
+      for (const component of state.components) {
+        statements.insertComponent.run(
+          component.projectId, component.projectSlug, component.versionNumber,
+          component.componentSetRevision, component.componentChecksum, component.instanceId,
+          component.templateId, component.instanceRevision, component.dataHash, canonicalJson(component.data)
+        );
+      }
+      for (const node of state.nodes) {
+        const result = statements.insertNode.run(
+          node.matchId, node.projectId, node.projectSlug, node.versionNumber,
+          node.componentSetRevision, node.componentChecksum, node.instanceId, node.templateId,
+          node.instanceRevision, node.dataHash,
+          node.pointer, node.field, node.valueType, node.valueJson, node.valueText,
+          node.searchText, node.foldedText
+        );
+        const nodeId = Number(result.lastInsertRowid);
+        statements.insertFts?.run(nodeId, node.foldedText);
+        for (const trigram of trigramValues(node.foldedText)) statements.insertTrigram.run(trigram, nodeId);
+      }
+      database.exec("COMMIT");
+      fs.chmodSync(storage.filePath, 0o600);
+    } catch (error) {
+      try { database.exec("ROLLBACK"); } catch (rollbackError) { void rollbackError; }
+      throw error;
+    }
+  }
+
+  function metadata() {
+    return rowsToMetadata(statements.metadata.all());
+  }
+
+  function queryRows(sql, parameters) {
+    return database.prepare(sql).all(...parameters);
+  }
+
+  function getMatch(matchId) {
+    return database.prepare("SELECT * FROM scalar_nodes WHERE match_id = ?").get(String(matchId || "")) || null;
+  }
+
+  function getComponent(filters) {
+    const where = [];
+    const parameters = [];
+    appendSqlIdentityFilters(where, parameters, filters, "components");
+    return queryRows(`SELECT * FROM components${where.length ? ` WHERE ${where.join(" AND ")}` : ""} ORDER BY project_slug, version_number, instance_id`, parameters);
+  }
+
+  function counts() {
+    const row = statements.counts.get();
+    return { components: Number(row?.components || 0), scalars: Number(row?.scalars || 0) };
+  }
+
+  function close() {
+    database.close();
+  }
+
+  return {
+    kind: "sqlite",
+    searchEngine: ftsEnabled ? "sqlite-fts5-trigram" : "sqlite-trigram",
+    ftsEnabled,
+    ...storage,
+    replaceState,
+    metadata,
+    queryRows,
+    getMatch,
+    getComponent,
+    counts,
+    close
+  };
+}
+
+function createStorageBackend(options, requestedBackend) {
+  if (requestedBackend === "json") return createJsonBackend(options);
+  try {
+    return createSqliteBackend(options);
+  } catch (error) {
+    if (requestedBackend === "sqlite" || error?.code !== "E_LOCAL_INDEX_SQLITE_UNAVAILABLE") throw error;
+    return createJsonBackend({ ...options, filePath: options.filePath && String(options.filePath).replace(/\.sqlite3?$/u, ".json") });
+  }
+}
+
+module.exports = Object.freeze({ createStorageBackend });

package/tools/writer-cli/src/local-index.cjs

@@ -0,0 +1,199 @@
+"use strict";
+
+const { cliError } = require("./errors.cjs");
+const {
+  INDEX_SCHEMA_VERSION,
+  SERVER_SNAPSHOT_SOURCE,
+  cloneJson,
+  flattenJsonScalars,
+  escapeJsonPointerToken,
+  unescapeJsonPointerToken,
+  normalizeSnapshot
+} = require("./local-index-snapshot.cjs");
+const { createStorageBackend } = require("./local-index-storage.cjs");
+const { purgeBackendFiles } = require("./local-index-cache.cjs");
+const {
+  createQueryLimits,
+  normalizeFilters,
+  appendSqlNodeFilters,
+  identityMatches,
+  normalizeSearch,
+  appendSqlSearch,
+  nodeMatchesSearch
+} = require("./local-index-query.cjs");
+const { rowToNode, freshnessFor, publicMatch, componentFromRow } = require("./local-index-lookup.cjs");
+
+function createLocalIndex(options = {}) {
+  const requestedBackend = String(options.backend || "auto");
+  if (!["auto", "sqlite", "json"].includes(requestedBackend)) {
+    throw cliError("E_LOCAL_INDEX_CONFIGURATION", `지원하지 않는 local index backend입니다: ${requestedBackend}`);
+  }
+  const backend = createStorageBackend(options, requestedBackend);
+  const limits = createQueryLimits(options, backend.ftsEnabled);
+  const nowProvider = typeof options.now === "function" ? options.now : Date.now;
+  let closed = false;
+
+  function assertOpen() {
+    if (closed) throw cliError("E_LOCAL_INDEX_CLOSED", "닫힌 로컬 인덱스입니다.");
+  }
+
+  function responseBase(metadata, freshnessOptions = {}) {
+    return {
+      authoritative: false,
+      source: "local-disposable-read-index",
+      requiresServerReadBeforeWrite: true,
+      backend: backend.kind,
+      searchEngine: backend.searchEngine,
+      freshness: freshnessFor(metadata, freshnessOptions, nowProvider)
+    };
+  }
+
+  function replaceSnapshot(snapshot) {
+    assertOpen();
+    const state = normalizeSnapshot(snapshot);
+    backend.replaceState(state);
+    return Object.freeze({
+      ...responseBase(state.metadata),
+      snapshotHash: state.metadata.snapshotHash,
+      projectVersionCount: state.metadata.projectVersionCount,
+      componentCount: state.metadata.componentCount,
+      scalarCount: state.metadata.scalarCount
+    });
+  }
+
+  function search(request = {}) {
+    assertOpen();
+    const metadata = backend.metadata();
+    if (!metadata) {
+      return Object.freeze({
+        ...responseBase(null, request),
+        total: 0,
+        returned: 0,
+        truncated: false,
+        matches: Object.freeze([])
+      });
+    }
+    const normalized = normalizeSearch(request, limits);
+    const candidateLimit = normalized.mode === "regex" ? limits.maxRegexScanRows : limits.maxCandidateRows;
+    let rawRows;
+    if (backend.kind === "sqlite") {
+      const where = [];
+      const parameters = [];
+      appendSqlNodeFilters(where, parameters, normalized.filters);
+      appendSqlSearch(where, parameters, normalized, limits);
+      rawRows = backend.queryRows(`SELECT scalar_nodes.* FROM scalar_nodes${where.length ? ` WHERE ${where.join(" AND ")}` : ""}
+        ORDER BY project_slug, version_number, instance_id, pointer LIMIT ?`, [...parameters, candidateLimit + 1]);
+    } else {
+      rawRows = backend.queryRows().map(rowToNode).filter(node => identityMatches(node, normalized.filters));
+      if (normalized.mode !== "regex") rawRows = rawRows.filter(node => nodeMatchesSearch(node, normalized));
+    }
+    let scanTruncated = rawRows.length > candidateLimit;
+    if (scanTruncated) rawRows = rawRows.slice(0, candidateLimit);
+    const matches = [];
+    let total = 0;
+    let omittedOversizedScalars = 0;
+    for (const row of rawRows) {
+      const node = rowToNode(row);
+      if (normalized.mode === "regex" && node.valueText.length > limits.maxRegexValueLength) {
+        omittedOversizedScalars += 1;
+        continue;
+      }
+      normalized.regex?.lastIndex && (normalized.regex.lastIndex = 0);
+      if (!nodeMatchesSearch(node, normalized)) continue;
+      total += 1;
+      if (matches.length < normalized.limit) matches.push(publicMatch(node));
+    }
+    const truncated = scanTruncated || omittedOversizedScalars > 0 || total > matches.length;
+    return Object.freeze({
+      ...responseBase(metadata, request),
+      total,
+      returned: matches.length,
+      truncated,
+      scanTruncated,
+      omittedOversizedScalars,
+      matches: Object.freeze(matches)
+    });
+  }
+
+  function getMatch(matchId, freshnessOptions = {}) {
+    assertOpen();
+    const metadata = backend.metadata();
+    const node = rowToNode(backend.getMatch(String(matchId || "")));
+    return Object.freeze({ ...responseBase(metadata, freshnessOptions), match: node ? publicMatch(node) : null });
+  }
+
+  function getComponent(request = {}) {
+    assertOpen();
+    const metadata = backend.metadata();
+    const filters = normalizeFilters(request);
+    const rows = backend.getComponent(filters).map(componentFromRow).filter(row => identityMatches({
+      ...row,
+      field: "",
+      pointer: "",
+      valueType: ""
+    }, { ...filters, fields: [], valueTypes: [], pathFilters: [] }));
+    return Object.freeze({
+      ...responseBase(metadata, request),
+      components: Object.freeze(rows.map(row => Object.freeze({
+        project: Object.freeze({ id: row.projectId, slug: row.projectSlug }),
+        versionNumber: row.versionNumber,
+        componentSet: Object.freeze({ revision: row.componentSetRevision, checksum: row.componentChecksum }),
+        instanceId: row.instanceId,
+        templateId: row.templateId,
+        revision: row.instanceRevision,
+        dataHash: row.dataHash,
+        data: cloneJson(row.data)
+      })))
+    });
+  }
+
+  function status(freshnessOptions = {}) {
+    assertOpen();
+    const metadata = backend.metadata();
+    return Object.freeze({
+      ...responseBase(metadata, freshnessOptions),
+      ready: Boolean(metadata),
+      filePath: backend.filePath,
+      snapshotHash: metadata?.snapshotHash || null,
+      actor: metadata?.actor || null,
+      projectVersionCount: metadata?.projectVersionCount || 0,
+      ...backend.counts()
+    });
+  }
+
+  function close() {
+    if (closed) return;
+    backend.close();
+    closed = true;
+  }
+
+  function purge() {
+    if (!closed) close();
+    const removedFiles = purgeBackendFiles(backend);
+    return Object.freeze({ authoritative: false, purged: true, removedFiles, filePath: backend.filePath });
+  }
+
+  return Object.freeze({
+    backend: backend.kind,
+    searchEngine: backend.searchEngine,
+    filePath: backend.filePath,
+    authoritative: false,
+    requiresServerReadBeforeWrite: true,
+    replaceSnapshot,
+    search,
+    getMatch,
+    getComponent,
+    status,
+    close,
+    purge
+  });
+}
+
+module.exports = Object.freeze({
+  INDEX_SCHEMA_VERSION,
+  SERVER_SNAPSHOT_SOURCE,
+  createLocalIndex,
+  flattenJsonScalars,
+  escapeJsonPointerToken,
+  unescapeJsonPointerToken
+});