larkit 0.0.0-stage → 0.1.0

package/src/indexer/format.js

@@ -0,0 +1,239 @@
+import fs from "node:fs";
+import path from "node:path";
+import {
+  ELECTRON_INDEX,
+  FORMAT_MANIFEST,
+  FORMATTED_DIR,
+  WEB_INDEX,
+} from "../shared/paths.js";
+import {
+  ensureDir,
+  fileSizeMiB,
+  log,
+  readJson,
+  safeJoin,
+  sha1,
+  toPosix,
+  writeJson,
+} from "../shared/util.js";
+import { runFormatPool } from "./format-pool.js";
+
+function parserFor(file) {
+  const ext = path.extname(file).toLowerCase();
+  if (ext === ".json") return "json";
+  if (ext === ".css") return "css";
+  return "babel";
+}
+
+function outputRel(entry) {
+  if (entry.scope === "web") return toPosix(path.join("web", entry.relPath));
+  if (entry.sourceTag === "loose") {
+    return toPosix(path.join("electron", "loose", entry.relPath));
+  }
+  const archiveBase = (entry.archive || entry.sourceTag || "asar").replace(
+    /\.asar$/,
+    "",
+  );
+  return toPosix(
+    path.join("electron", "asar", archiveBase, entry.relPath),
+  );
+}
+
+function collectEntries() {
+  const web = readJson(WEB_INDEX, null);
+  const electron = readJson(ELECTRON_INDEX, null);
+  if (!web?.entries?.length) {
+    throw new Error(`missing ${WEB_INDEX}; run extract first`);
+  }
+  if (!electron?.entries?.length) {
+    throw new Error(`missing ${ELECTRON_INDEX}; run extract first`);
+  }
+  const entries = [];
+  for (const entry of [...web.entries, ...electron.entries]) {
+    if (!fs.existsSync(entry.sourcePath)) continue;
+    entries.push({
+      ...entry,
+      outRel: outputRel(entry),
+      parser: parserFor(entry.relPath),
+    });
+  }
+  return entries;
+}
+
+export async function formatAssets({ force = false, concurrency, limit } = {}) {
+  if (force && fs.existsSync(FORMATTED_DIR)) {
+    fs.rmSync(FORMATTED_DIR, { recursive: true, force: true });
+  }
+  ensureDir(FORMATTED_DIR);
+
+  const entries = collectEntries();
+  const manifest = force
+    ? { version: 2, entries: {} }
+    : readJson(FORMAT_MANIFEST, { version: 2, entries: {} });
+  if (!manifest.entries) manifest.entries = {};
+
+  const cacheHits = [];
+  const pending = [];
+  for (const entry of entries) {
+    let srcHash;
+    try {
+      srcHash = sha1(fs.readFileSync(entry.sourcePath));
+    } catch (error) {
+      log(`[format] skip unreadable ${entry.sourcePath}: ${error.message}`);
+      continue;
+    }
+    const outPath = safeJoin(FORMATTED_DIR, entry.outRel);
+    const cached = manifest.entries[entry.outRel];
+    if (
+      !force &&
+      cached &&
+      cached.srcHash === srcHash &&
+      fs.existsSync(outPath) &&
+      fs.statSync(outPath).size > 0
+    ) {
+      cacheHits.push(entry);
+      continue;
+    }
+    pending.push({ ...entry, srcHash, outPath });
+  }
+
+  log(
+    `[format] ${entries.length} sources: ${cacheHits.length} cached, ${pending.length} to format`,
+  );
+
+  // Content-addressed dedupe: many asar chunks are byte-identical. Format one
+  // representative, then copy the formatted text to every alias.
+  const groups = new Map();
+  for (const entry of pending) {
+    const key = `${entry.srcHash}:${entry.parser}`;
+    let group = groups.get(key);
+    if (!group) {
+      group = { key, parser: entry.parser, members: [] };
+      groups.set(key, group);
+    }
+    group.members.push(entry);
+  }
+
+  const groupList = [...groups.values()];
+  const limitedGroups = limit ? groupList.slice(0, limit) : groupList;
+  const taskById = new Map();
+  const tasks = limitedGroups.map((group, index) => {
+    const id = `g${index}`;
+    taskById.set(id, group);
+    return {
+      id,
+      srcPath: group.members[0].sourcePath,
+      parser: group.parser,
+    };
+  });
+
+  const stats = {
+    cached: cacheHits.length,
+    formattedFiles: 0,
+    formattedBytes: 0,
+    lines: 0,
+    fallback: 0,
+    errors: 0,
+    deduped: 0,
+  };
+  let completedGroups = 0;
+
+  if (tasks.length > 0) {
+    await runFormatPool(tasks, {
+      concurrency,
+      progress: 100,
+      onResult: (result) => {
+        completedGroups++;
+        const group = taskById.get(result.id);
+        if (!group) return;
+        const representative = group.members[0];
+        if (!result.ok) {
+          stats.errors++;
+          log(
+            `[format] failed ${representative.outRel}: ${result.error}`,
+          );
+          return;
+        }
+        ensureDir(path.dirname(representative.outPath));
+        fs.writeFileSync(representative.outPath, result.code);
+        manifest.entries[representative.outRel] = {
+          srcHash: representative.srcHash,
+          srcBytes: result.srcBytes,
+          outBytes: result.outBytes,
+          lines: result.lines,
+          parser: result.parser,
+          fallback: result.fallback,
+        };
+        stats.formattedFiles++;
+        stats.formattedBytes += result.outBytes;
+        stats.lines += result.lines;
+        if (result.fallback) stats.fallback++;
+
+        for (const alias of group.members.slice(1)) {
+          ensureDir(path.dirname(alias.outPath));
+          fs.copyFileSync(representative.outPath, alias.outPath);
+          manifest.entries[alias.outRel] = {
+            srcHash: alias.srcHash,
+            srcBytes: result.srcBytes,
+            outBytes: result.outBytes,
+            lines: result.lines,
+            parser: result.parser,
+            fallback: result.fallback,
+            copiedFrom: representative.outRel,
+          };
+          stats.deduped++;
+        }
+      },
+    });
+  }
+
+  // Remove manifest entries whose source disappeared.
+  const live = new Set(entries.map((e) => e.outRel));
+  for (const key of Object.keys(manifest.entries)) {
+    if (!live.has(key)) delete manifest.entries[key];
+  }
+  manifest.version = 2;
+  manifest.generatedAt = new Date().toISOString();
+  manifest.sources = entries.length;
+  manifest.stats = stats;
+  writeJson(FORMAT_MANIFEST, manifest);
+
+  const allOutputs = entries.filter((e) =>
+    fs.existsSync(safeJoin(FORMATTED_DIR, e.outRel)),
+  );
+  const totalBytes = allOutputs.reduce((sum, e) => {
+    try {
+      return sum + fs.statSync(safeJoin(FORMATTED_DIR, e.outRel)).size;
+    } catch {
+      return sum;
+    }
+  }, 0);
+
+  const readme = `# Formatted code corpus
+
+Generated by \`larkit index:format\`.
+
+- Sources: ${entries.length}
+- Outputs: ${allOutputs.length}
+- Formatted bytes: ${fileSizeMiB(totalBytes)}
+- Cache hits: ${stats.cached}
+- Newly formatted: ${stats.formattedFiles}
+- Deduplicated aliases: ${stats.deduped}
+- Parser fallbacks: ${stats.fallback}
+- Errors: ${stats.errors}
+
+Layout:
+
+- \`web/<debugger|stylesheet|network>/...\` — 531 deduplicated web-client resources.
+- \`electron/loose/...\` — JS/CSS/JSON already present in the deb's webcontent.
+- \`electron/asar/<archive>/...\` — non-i18n JS/CSS/JSON extracted from desktop asars.
+
+Every file is multi-line formatted; line numbers in \`truth.db\` refer to these
+files.  Use \`larkit inspect <file> <line>\` for a bounded slice.
+`;
+  fs.writeFileSync(path.join(FORMATTED_DIR, "README.md"), readme);
+  log(
+    `[format] done: ${allOutputs.length} outputs, ${fileSizeMiB(totalBytes)}, ${stats.errors} errors, ${stats.deduped} aliases`,
+  );
+  return { entries, stats, totalBytes, outputs: allOutputs.length };
+}

package/src/indexer/protocol.js

@@ -0,0 +1,123 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  PROTOCOL_CATALOG_JSON,
+  PROTOCOL_DOCS_JSON,
+  ROOT,
+} from "../shared/paths.js";
+import { log, readJson, relativeTo, truncate, writeJson } from "../shared/util.js";
+import { loadCatalog } from "../protocol/catalog.js";
+import { createRegistry } from "../protocol/schema.js";
+
+const REFERENCE_DIR = fileURLToPath(
+  new URL("../protocol/data/reference", import.meta.url),
+);
+
+function schemaSummary(registry, typeName) {
+  if (!typeName) return "";
+  try {
+    const info = registry.typeInfo(typeName);
+    if (!info) return `${typeName}: <schema unavailable>`;
+    if (info.kind === "enum") {
+      const values = (info.enumValues || [])
+        .slice(0, 30)
+        .map((value) => `${value.name}=${value.value}`)
+        .join(", ");
+      return truncate(`${info.fq}{${values}}`, 1000);
+    }
+    const fields = (info.fields || [])
+      .slice(0, 40)
+      .map((field) => {
+        const suffix =
+          field.rule === "repeated" ? "[]" : field.keyType ? "{}" : "";
+        return `${field.name}:${field.type}${suffix}`;
+      })
+      .join(", ");
+    return truncate(`${info.fq}{${fields}}`, 1400);
+  } catch (error) {
+    return `${typeName}: ${error.message}`;
+  }
+}
+
+function splitMarkdownSections(file, repoPath) {
+  const text = fs.readFileSync(file, "utf8");
+  const lines = text.split("\n");
+  const sections = [];
+  let current = null;
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.startsWith("## ")) {
+      if (current) sections.push(current);
+      current = {
+        title: line.slice(3).trim(),
+        path: repoPath,
+        line_start: i + 1,
+        line_end: lines.length,
+        body: [],
+      };
+    }
+    if (current) current.body.push(line);
+  }
+  if (current) sections.push(current);
+  for (const section of sections) {
+    section.line_end = section.line_start + section.body.length - 1;
+    section.body = section.body.join("\n");
+  }
+  return sections;
+}
+
+export async function buildProtocolAssets() {
+  const registry = createRegistry();
+  const { entries: apis } = loadCatalog();
+  const catalog = apis.map((api) => {
+    const requestSchema = schemaSummary(registry, api.requestType);
+    const responseSchema = schemaSummary(registry, api.responseType);
+    return {
+      cmd: api.cmd,
+      name: api.name || "",
+      request_type: api.requestType || "",
+      response_type: api.responseType || "",
+      kind: api.kind,
+      version: api.version ?? null,
+      schema_available: api.schemaAvailable ? 1 : 0,
+      source: api.source || "",
+      schema_summary: [requestSchema, responseSchema].filter(Boolean).join("\n"),
+    };
+  });
+  writeJson(PROTOCOL_CATALOG_JSON, {
+    generatedAt: new Date().toISOString(),
+    total: catalog.length,
+    entries: catalog,
+  });
+
+  const docs = [];
+  const protocolMd = path.join(REFERENCE_DIR, "PROTOCOL.md");
+  if (fs.existsSync(protocolMd)) {
+    docs.push(
+      ...splitMarkdownSections(protocolMd, relativeTo(ROOT, protocolMd)),
+    );
+  }
+  const endpointsJson = path.join(REFERENCE_DIR, "endpoints.json");
+  if (fs.existsSync(endpointsJson)) {
+    const data = readJson(endpointsJson, {});
+    for (const [key, value] of Object.entries(data)) {
+      docs.push({
+        title: `endpoints.json#${key}`,
+        path: relativeTo(ROOT, endpointsJson),
+        line_start: 1,
+        line_end: 1,
+        body: typeof value === "string" ? value : JSON.stringify(value, null, 2),
+      });
+    }
+  }
+  writeJson(PROTOCOL_DOCS_JSON, {
+    generatedAt: new Date().toISOString(),
+    total: docs.length,
+    entries: docs,
+  });
+  log(
+    `[protocol] ${catalog.length} catalog entries, ${docs.length} protocol doc sections`,
+  );
+  return { catalog, docs };
+}