larkit 0.0.0-stage → 0.1.0

package/src/indexer/extract-assets.js

@@ -0,0 +1,285 @@
+import fs from "node:fs";
+import path from "node:path";
+import * as asar from "@electron/asar";
+import {
+  ELECTRON_EXTRACT_DIR,
+  ELECTRON_INDEX,
+  FEISHU_APP_DIR,
+  RAW_DIR,
+  WEB_DUMP_DIR,
+  WEB_INDEX,
+  WEB_SUMMARY,
+} from "../shared/paths.js";
+import {
+  ensureDir,
+  fileSizeMiB,
+  log,
+  readJson,
+  relativeTo,
+  toPosix,
+  writeJson,
+} from "../shared/util.js";
+import { ensureDebExtracted, directorySize } from "./extract-deb.js";
+
+/**
+ * Curated desktop asar set.  The Linux client has no single app.asar: its main
+ * process is native (feishu/libiron), and every window is a separate asar.
+ * We extract all non-i18n JS from the windows that carry messaging, cards,
+ * spaces and settings, plus the preload/resource bundles.
+ */
+const ASAR_SELECTION = [
+  { name: "resource.asar", all: true },
+  { name: "lark-api.asar", all: true },
+  { name: "jssdk-mf.asar", all: true },
+  { name: "js-worker.asar", all: true },
+  { name: "shared-libs.asar", all: true },
+  { name: "messenger.asar", all: false },
+  { name: "main-window.asar", all: false },
+  { name: "messenger-modals.asar", all: false },
+  { name: "space.asar", all: false },
+  { name: "setting.asar", all: false },
+  { name: "search.asar", all: false },
+];
+
+const LOOSE_EXTENSIONS = new Set([".js", ".css", ".json"]);
+const ASAR_EXTENSIONS = new Set([".js", ".css", ".json"]);
+
+function isInterestingAsarEntry(entry, all) {
+  if (entry.endsWith("/")) return false;
+  const ext = path.extname(entry).toLowerCase();
+  if (!ASAR_EXTENSIONS.has(ext)) return false;
+  if (!all && entry.startsWith("/i18n/")) return false;
+  // Massive locale bundles are noise for a code/truth index.
+  if (entry.includes("/i18n/") && !all) return false;
+  return true;
+}
+
+function buildWebIndex() {
+  const summary = readJson(WEB_SUMMARY, null);
+  if (!summary?.captures?.length) {
+    throw new Error(`missing or invalid ${WEB_SUMMARY}`);
+  }
+  const entries = [];
+  const seen = new Set();
+  let missing = 0;
+  let bytes = 0;
+  for (const capture of summary.captures) {
+    const key = capture.file;
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    const sourcePath = path.join(WEB_DUMP_DIR, capture.file);
+    if (!fs.existsSync(sourcePath)) {
+      missing++;
+      continue;
+    }
+    const stat = fs.statSync(sourcePath);
+    bytes += stat.size;
+    entries.push({
+      scope: "web",
+      sourceTag: "web",
+      relPath: toPosix(capture.file),
+      sourcePath,
+      assetType: capture.assetType,
+      kind: capture.kind,
+      sha1: capture.sha1,
+      bytes: stat.size,
+      resourceUrl: capture.resourceUrl,
+      targetType: capture.targetType,
+    });
+  }
+  writeJson(WEB_INDEX, {
+    generatedAt: new Date().toISOString(),
+    totalCaptures: summary.captures.length,
+    uniqueFiles: entries.length,
+    missing,
+    bytes,
+    entries,
+  });
+  log(
+    `[extract] web index: ${entries.length}/${summary.captures.length} files, ${fileSizeMiB(bytes)}${missing ? `, missing=${missing}` : ""}`,
+  );
+  return entries;
+}
+
+function buildLooseElectronIndex() {
+  const root = path.join(FEISHU_APP_DIR, "webcontent");
+  const entries = [];
+  const seen = new Set();
+  const stack = [root];
+  while (stack.length) {
+    const current = stack.pop();
+    let children;
+    try {
+      children = fs.readdirSync(current, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const child of children) {
+      if (child.name.startsWith(".")) continue;
+      const full = path.join(current, child.name);
+      if (child.isDirectory()) {
+        if (child.name === "i18n" || child.name === "locales") continue;
+        if (child.name === "node_modules") continue;
+        stack.push(full);
+        continue;
+      }
+      if (!child.isFile()) continue;
+      const ext = path.extname(child.name).toLowerCase();
+      if (!LOOSE_EXTENSIONS.has(ext)) continue;
+      let stat;
+      try {
+        stat = fs.statSync(full);
+      } catch {
+        continue;
+      }
+      if (stat.size === 0 || stat.size > 15 * 1024 * 1024) continue;
+      const rel = relativeTo(root, full);
+      if (seen.has(rel)) continue;
+      seen.add(rel);
+      entries.push({
+        scope: "electron",
+        sourceTag: "loose",
+        relPath: rel,
+        sourcePath: full,
+        assetType: ext.slice(1),
+        bytes: stat.size,
+      });
+    }
+  }
+  return entries;
+}
+
+function buildAsarElectronIndex() {
+  const webcontent = path.join(FEISHU_APP_DIR, "webcontent");
+  const entries = [];
+  let totalBytes = 0;
+  let extracted = 0;
+  let skipped = 0;
+  for (const selection of ASAR_SELECTION) {
+    const archive = path.join(webcontent, selection.name);
+    if (!fs.existsSync(archive)) {
+      log(`[extract] asar missing: ${selection.name}`);
+      continue;
+    }
+    const archiveBase = selection.name.replace(/\.asar$/, "");
+    const outRoot = path.join(ELECTRON_EXTRACT_DIR, "asar", archiveBase);
+    ensureDir(outRoot);
+    const list = asar.listPackage(archive);
+    let archiveCount = 0;
+    for (const entry of list) {
+      if (!isInterestingAsarEntry(entry, selection.all)) continue;
+      const rel = entry.replace(/^\/+/, "");
+      const dest = path.join(outRoot, rel);
+      const ext = path.extname(rel).toLowerCase();
+      archiveCount++;
+      if (fs.existsSync(dest) && fs.statSync(dest).size > 0) {
+        skipped++;
+      } else {
+        try {
+          const buf = asar.extractFile(archive, rel);
+          ensureDir(path.dirname(dest));
+          fs.writeFileSync(dest, buf);
+          extracted++;
+        } catch (error) {
+          log(`[extract] asar read failed ${selection.name}:${entry}: ${error.message}`);
+          continue;
+        }
+      }
+      let size = 0;
+      try {
+        size = fs.statSync(dest).size;
+      } catch {
+        continue;
+      }
+      totalBytes += size;
+      entries.push({
+        scope: "electron",
+        sourceTag: `asar/${archiveBase}`,
+        relPath: rel,
+        sourcePath: dest,
+        assetType: ext.slice(1),
+        bytes: size,
+        archive: selection.name,
+      });
+    }
+    log(
+      `[extract] ${selection.name}: ${archiveCount} selected files (${extracted} new, ${skipped} cached)`,
+    );
+  }
+  return { entries, totalBytes, extracted, skipped };
+}
+
+function writeElectronReadme() {
+  const file = path.join(ELECTRON_EXTRACT_DIR, "README.md");
+  const text = `# Electron extraction notes
+
+The Linux client **does not ship a single \`app.asar\`**.  Its window code lives in
+\`webcontent/*.asar\`, while the Electron main process is the native \`feishu\`
+binary plus \`libiron\` / \`liblark\`.  This directory therefore contains:
+
+- \`loose/\`: JS/CSS/JSON that the deb already extracts under \`webcontent/\`
+  (jssdk worklets, js-worker, openplatform, lark-api, bridge.js, ...).
+- \`asar/<archive>/\`: every non-i18n JS/CSS/JSON entry from the curated asar
+  set (messenger, main-window, messenger-modals, space, setting, search,
+  resource, lark-api, jssdk-mf, js-worker, shared-libs).
+
+The \`resource.asar/js/*\` files are the closest equivalent to preload /
+bootstrap scripts; \`messenger.asar/common/*\` and
+\`main-window.asar/common/*\` are the renderer chunks used by the desktop UI.
+
+No package.json was present inside the selected asar archives.  Desktop
+protocol/schema truth ultimately comes from \`pb.desc\`, \`pb_server.desc\`,
+the jssdk worklets and these renderer chunks.
+`;
+  fs.writeFileSync(file, text);
+}
+
+export function extractAssets() {
+  ensureDebExtracted();
+  ensureDir(RAW_DIR);
+  ensureDir(ELECTRON_EXTRACT_DIR);
+  writeElectronReadme();
+
+  const webEntries = buildWebIndex();
+  const looseEntries = buildLooseElectronIndex();
+  const asarResult = buildAsarElectronIndex();
+  const electronEntries = [...looseEntries, ...asarResult.entries];
+
+  writeJson(ELECTRON_INDEX, {
+    generatedAt: new Date().toISOString(),
+    loose: looseEntries.length,
+    asar: asarResult.entries.length,
+    total: electronEntries.length,
+    bytes: electronEntries.reduce((sum, e) => sum + e.bytes, 0),
+    entries: electronEntries,
+  });
+
+  const manifest = {
+    generatedAt: new Date().toISOString(),
+    deb: "feishu-linux.deb",
+    clientVersion: readClientVersion(),
+    web: { entries: webEntries.length, bytes: webEntries.reduce((s, e) => s + e.bytes, 0) },
+    electron: {
+      loose: looseEntries.length,
+      asar: asarResult.entries.length,
+      extractedNow: asarResult.extracted,
+      cached: asarResult.skipped,
+      bytes: electronEntries.reduce((s, e) => s + e.bytes, 0),
+    },
+    extractedDirBytes: directorySize(ELECTRON_EXTRACT_DIR),
+  };
+  writeJson(path.join(RAW_DIR, "manifest.extract.json"), manifest);
+  log(
+    `[extract] electron index: ${electronEntries.length} files (${manifest.electron.bytes ? fileSizeMiB(manifest.electron.bytes) : "0"})`,
+  );
+  return { webEntries, electronEntries, manifest };
+}
+
+function readClientVersion() {
+  try {
+    const manifest = readJson(path.join(FEISHU_APP_DIR, "manifest.json"), {});
+    return manifest.appVersion || "unknown";
+  } catch {
+    return "unknown";
+  }
+}

package/src/indexer/extract-deb.js

@@ -0,0 +1,56 @@
+import fs from "node:fs";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { DEB_FILE, DEB_ROOT, FEISHU_APP_DIR } from "../shared/paths.js";
+import { ensureDir, log, fileSizeMiB } from "../shared/util.js";
+
+export function ensureDebExtracted({ force = false } = {}) {
+  const marker = path.join(FEISHU_APP_DIR, "manifest.json");
+  if (!force && fs.existsSync(marker)) {
+    log(`[extract] deb already extracted: ${DEB_ROOT}`);
+    return FEISHU_APP_DIR;
+  }
+  if (!fs.existsSync(DEB_FILE)) {
+    throw new Error(`missing ${DEB_FILE}`);
+  }
+  ensureDir(DEB_ROOT);
+  log(`[extract] dpkg-deb -x ${path.basename(DEB_FILE)} -> ${DEB_ROOT}`);
+  const t0 = Date.now();
+  const result = spawnSync("dpkg-deb", ["-x", DEB_FILE, DEB_ROOT], {
+    stdio: "inherit",
+  });
+  if (result.status !== 0) {
+    throw new Error(`dpkg-deb failed with exit code ${result.status}`);
+  }
+  const size = directorySize(DEB_ROOT);
+  log(
+    `[extract] deb extracted in ${((Date.now() - t0) / 1000).toFixed(1)}s (${fileSizeMiB(size)})`,
+  );
+  return FEISHU_APP_DIR;
+}
+
+export function directorySize(dir) {
+  let total = 0;
+  const stack = [dir];
+  while (stack.length) {
+    const current = stack.pop();
+    let entries;
+    try {
+      entries = fs.readdirSync(current, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      const full = path.join(current, entry.name);
+      if (entry.isDirectory()) stack.push(full);
+      else if (entry.isFile()) {
+        try {
+          total += fs.statSync(full).size;
+        } catch {
+          // ignore
+        }
+      }
+    }
+  }
+  return total;
+}

package/src/indexer/format-pool.js

@@ -0,0 +1,64 @@
+import os from "node:os";
+import { Worker } from "node:worker_threads";
+
+export async function runFormatPool(tasks, options = {}) {
+  const concurrency = Math.max(
+    1,
+    Math.min(
+      options.concurrency || Math.min(8, os.availableParallelism?.() || os.cpus().length),
+      tasks.length || 1,
+    ),
+  );
+  if (tasks.length === 0) return [];
+
+  const workerUrl = new URL("./format-worker.js", import.meta.url);
+  const workers = [];
+  const results = [];
+  let next = 0;
+  let completed = 0;
+  let failed = false;
+
+  return await new Promise((resolve, reject) => {
+    const finish = () => {
+      for (const worker of workers) worker.terminate();
+      resolve(results);
+    };
+
+    const assign = (worker) => {
+      if (next >= tasks.length) {
+        if (completed === tasks.length && !failed) finish();
+        return;
+      }
+      const task = tasks[next++];
+      worker.postMessage(task);
+    };
+
+    const onMessage = (worker, msg) => {
+      completed++;
+      results.push({ ...msg, code: undefined });
+      if (typeof options.onResult === "function") options.onResult(msg);
+      if (options.progress && completed % options.progress === 0) {
+        console.error(`[format] ${completed}/${tasks.length}`);
+      }
+      assign(worker);
+    };
+
+    for (let i = 0; i < concurrency; i++) {
+      const worker = new Worker(workerUrl);
+      workers.push(worker);
+      worker.on("message", (msg) => onMessage(worker, msg));
+      worker.on("error", (error) => {
+        failed = true;
+        for (const w of workers) w.terminate();
+        reject(error);
+      });
+      worker.on("exit", (code) => {
+        if (code !== 0 && !failed) {
+          failed = true;
+          reject(new Error(`format worker exited with code ${code}`));
+        }
+      });
+      assign(worker);
+    }
+  });
+}

package/src/indexer/format-worker.js

@@ -0,0 +1,132 @@
+import fs from "node:fs";
+import path from "node:path";
+import { parentPort } from "node:worker_threads";
+import * as prettier from "prettier";
+import beautify from "js-beautify";
+
+function parserFor(file) {
+  const ext = path.extname(file).toLowerCase();
+  if (ext === ".json") return "json";
+  if (ext === ".css") return "css";
+  if (ext === ".html") return "html";
+  return "babel";
+}
+
+function basicBreak(code) {
+  // Last-resort line breaker for pathological minified input.  It keeps strings
+  // intact and inserts newlines at structural characters.
+  let out = "";
+  let quote = "";
+  let escape = false;
+  for (let i = 0; i < code.length; i++) {
+    const ch = code[i];
+    out += ch;
+    if (quote) {
+      if (escape) escape = false;
+      else if (ch === "\\") escape = true;
+      else if (ch === quote) quote = "";
+      continue;
+    }
+    if (ch === '"' || ch === "'" || ch === "`") {
+      quote = ch;
+      continue;
+    }
+    if (ch === ";" || ch === "{" || ch === "}") {
+      if (i + 1 < code.length && code[i + 1] !== "\n") out += "\n";
+    }
+  }
+  return out;
+}
+
+function beautifyFallback(code, parser) {
+  try {
+    if (parser === "json") return `${JSON.stringify(JSON.parse(code), null, 2)}\n`;
+    if (parser === "css") {
+      return `${beautify.css(code, { indent_size: 2, preserve_newlines: false })}\n`;
+    }
+    if (parser === "html") {
+      return `${beautify.html(code, { indent_size: 2, wrap_line_length: 120 })}\n`;
+    }
+    return `${beautify.js(code, {
+      indent_size: 2,
+      preserve_newlines: false,
+      max_preserve_newlines: 1,
+      brace_style: "collapse,preserve-inline",
+      end_with_newline: true,
+    })}\n`;
+  } catch {
+    return `${basicBreak(code)}\n`;
+  }
+}
+
+parentPort.on("message", async (task) => {
+  const { id, srcPath, parser: requestedParser } = task;
+  const parser = requestedParser || parserFor(srcPath);
+  try {
+    const buf = fs.readFileSync(srcPath);
+    if (buf.includes(0)) {
+      throw new Error("binary content");
+    }
+    const code = buf.toString("utf8");
+    if (!code.trim()) {
+      throw new Error("empty content");
+    }
+
+    let out = "";
+    let used = parser;
+    let fallback = false;
+
+    // js-beautify is substantially faster for huge bundles; Prettier gives the
+    // best line-number fidelity for the normal case.
+    if (buf.length > 4 * 1024 * 1024) {
+      fallback = true;
+      out = beautifyFallback(code, parser);
+    } else {
+      try {
+        out = await prettier.format(code, {
+          parser,
+          printWidth: 120,
+          tabWidth: 2,
+          useTabs: false,
+          semi: true,
+          singleQuote: false,
+          trailingComma: "all",
+          bracketSpacing: true,
+          arrowParens: "always",
+          endOfLine: "lf",
+        });
+        used = parser;
+      } catch (error) {
+        fallback = true;
+        out = beautifyFallback(code, parser);
+        out = `${out}`;
+        if (!out.endsWith("\n")) out += "\n";
+      }
+    }
+
+    if (buf.length > 500 && !out.includes("\n")) {
+      fallback = true;
+      out = beautifyFallback(out, parser);
+    }
+    if (!out.endsWith("\n")) out += "\n";
+
+    const lines = out.split("\n").length - 1;
+    parentPort.postMessage({
+      id,
+      ok: true,
+      code: out,
+      parser: used,
+      fallback,
+      srcBytes: buf.length,
+      outBytes: Buffer.byteLength(out),
+      lines,
+    });
+  } catch (error) {
+    parentPort.postMessage({
+      id,
+      ok: false,
+      error: error instanceof Error ? error.message : String(error),
+      parser,
+    });
+  }
+});