@functional-systems/srcx 0.1.0-dev.1

package/src/chunk.ts

@@ -0,0 +1,328 @@
+import { createRequire } from "node:module";
+import { Parser, Language, type Node } from "web-tree-sitter";
+import { getEncoding } from "js-tiktoken";
+import { invariant, lineAt } from "./common.js";
+const require = createRequire(import.meta.url);
+const encoding = getEncoding("cl100k_base");
+export function tokens(text: string): number {
+  return encoding.encode(text, [], []).length;
+}
+export const CHUNKER = {
+  version: 1,
+  parser: "web-tree-sitter@0.25.10",
+  grammars: "tree-sitter-wasms@0.1.13",
+  tokenizer: "js-tiktoken@1.0.21/cl100k_base",
+  targetTokens: 800,
+  maxTokens: 1500,
+  fallbackOverlap: 0.1,
+  enrichment: "path-scope-symbol-v1",
+  policy: "source-v1",
+} as const;
+export type Span = {
+  startByte: number;
+  endByte: number;
+  startLine: number;
+  endLine: number;
+  chunkKind: string;
+  scope?: string;
+  symbol?: string;
+  signature?: string;
+  searchText: string;
+  tokenCount: number;
+};
+type Unit = {
+  start: number;
+  end: number;
+  kind: string;
+  scope?: string;
+  symbol?: string;
+  signature?: string;
+};
+const languages = new Map<string, Language>();
+let initialized: Promise<void> | undefined;
+async function language(name: string): Promise<Language> {
+  initialized ??= Parser.init();
+  await initialized;
+  let l = languages.get(name);
+  if (!l) {
+    l = await Language.load(
+      require.resolve(`tree-sitter-wasms/out/tree-sitter-${name}.wasm`),
+    );
+    languages.set(name, l);
+  }
+  return l;
+}
+export function detectLanguage(path: string): string {
+  const ext = path.split(".").at(-1)?.toLowerCase();
+  return (
+    (
+      {
+        java: "java",
+        ts: "typescript",
+        tsx: "tsx",
+        js: "javascript",
+        jsx: "javascript",
+        mjs: "javascript",
+        cjs: "javascript",
+        md: "markdown",
+        mdx: "markdown",
+        json: "json",
+        yaml: "yaml",
+        yml: "yaml",
+        toml: "toml",
+        ini: "ini",
+      } as Record<string, string>
+    )[ext ?? ""] ?? "text"
+  );
+}
+function enrichment(path: string, u: Unit): string {
+  return (
+    [path.slice(0, 240), u.scope?.slice(0, 160), u.symbol?.slice(0, 160)]
+      .filter(Boolean)
+      .join("\n") + "\n"
+  );
+}
+function classify(type: string): string {
+  if (/import|package/.test(type)) return "imports";
+  if (/comment/.test(type)) return "documentation";
+  if (/function|method|constructor/.test(type)) return "function";
+  return "declaration";
+}
+function scopeName(n: Node): string | undefined {
+  const names: string[] = [];
+  let p = n.parent;
+  while (p) {
+    if (/class|interface|function|method/.test(p.type)) {
+      const name = p.childForFieldName("name");
+      if (name) names.unshift(name.text);
+    }
+    p = p.parent;
+  }
+  return names.length ? names.join(".") : undefined;
+}
+/** Bound strings without slicing a surrogate pair; prefer line ends where possible. */
+function splitUnit(
+  source: string,
+  path: string,
+  u: Unit,
+  overlap: number,
+): Unit[] {
+  const result: Unit[] = [];
+  let start = u.start;
+  const prefix = enrichment(path, u);
+  invariant(
+    tokens(prefix) < CHUNKER.maxTokens,
+    "Path/scope enrichment exceeds token limit.",
+  );
+  while (start < u.end) {
+    let end = u.end;
+    if (tokens(prefix + source.slice(start, end)) > CHUNKER.maxTokens) {
+      let lo = start + 1,
+        hi = end,
+        best = start;
+      while (lo <= hi) {
+        const probe = Math.floor((lo + hi) / 2);
+        let mid = probe;
+        if (mid < u.end && /[\uDC00-\uDFFF]/.test(source[mid] ?? "")) mid--;
+        if (mid <= start) {
+          lo = probe + 1;
+          continue;
+        }
+        if (tokens(prefix + source.slice(start, mid)) <= CHUNKER.targetTokens) {
+          best = mid;
+          lo = probe + 1;
+        } else hi = probe - 1;
+      }
+      invariant(best > start, "Cannot fit source unit in token budget.");
+      end = best;
+      const newline = source.lastIndexOf("\n", end - 1);
+      if (newline >= start + (end - start) / 2) end = newline + 1;
+    }
+    result.push({ ...u, start, end });
+    if (end === u.end) break;
+    let next = end;
+    if (overlap) {
+      next = Math.max(start + 1, end - Math.floor((end - start) * overlap));
+      if (/[\uDC00-\uDFFF]/.test(source[next] ?? "")) next++;
+    }
+    invariant(next > start, "Chunker made no progress.");
+    start = next;
+  }
+  return result;
+}
+function textUnits(source: string, lang: string): Unit[] {
+  // Heading/paragraph/fence units for Markdown; config section/line units otherwise.
+  const units: Unit[] = [];
+  let start = 0,
+    pos = 0,
+    fenced = false;
+  for (const line of source.match(/[^\n]*\n|[^\n]+$/g) ?? []) {
+    const fence = /^\s*(```|~~~)/.test(line);
+    const boundary =
+      lang === "markdown"
+        ? !fenced && (/^#{1,6}\s/.test(line) || line.trim() === "")
+        : /^\s*(\[|[\w.-]+\s*[:=])/.test(line);
+    if (boundary && pos > start) {
+      units.push({
+        start,
+        end: pos,
+        kind: lang === "markdown" ? "documentation" : "configuration",
+      });
+      start = pos;
+    }
+    if (fence) fenced = !fenced;
+    pos += line.length;
+  }
+  if (pos > start)
+    units.push({
+      start,
+      end: pos,
+      kind: lang === "markdown" ? "documentation" : "configuration",
+    });
+  return units;
+}
+export async function chunk(
+  source: string,
+  path: string,
+  mode: "syntax" | "window" = "syntax",
+): Promise<{ spans: Span[]; parseStatus: string; language: string }> {
+  const lang = detectLanguage(path);
+  if (!source.length)
+    return { spans: [], parseStatus: "empty", language: lang };
+  let units: Unit[] = [];
+  let status = "parsed";
+  if (mode === "window") {
+    status = "window-baseline";
+    units = [{ start: 0, end: source.length, kind: "fallback" }];
+  } else if (["java", "typescript", "tsx", "javascript"].includes(lang)) {
+    const grammar = await language(lang);
+    const parser = new Parser();
+    let tree: ReturnType<Parser["parse"]> = null;
+    try {
+      parser.setLanguage(grammar);
+      tree = parser.parse(source);
+      invariant(tree, "Parser returned no tree.");
+      if (tree.rootNode.hasError) {
+        status = "parse-error-fallback";
+        units = [{ start: 0, end: source.length, kind: "fallback" }];
+      } else {
+        function visit(n: Node): void {
+          const container =
+            /^(program|source_file|class_body|interface_body|class_declaration|interface_declaration|export_statement)$/.test(
+              n.type,
+            );
+          const u: Unit = {
+            start: n.startIndex,
+            end: n.endIndex,
+            kind: classify(n.type),
+            scope: scopeName(n),
+          };
+          const name = n.childForFieldName("name");
+          if (name) u.symbol = name.text.slice(0, 200);
+          if (u.kind === "function")
+            u.signature = n.text.split(/[\n{]/, 1)[0]?.slice(0, 240);
+          if (
+            n.namedChildCount &&
+            (container ||
+              tokens(enrichment(path, u) + n.text) > CHUNKER.maxTokens)
+          ) {
+            for (const child of n.namedChildren) if (child) visit(child);
+          } else if (n.endIndex > n.startIndex) units.push(u);
+        }
+        visit(tree.rootNode);
+        // Attach every gap (punctuation, BOM, comments, whitespace) without altering source bytes.
+        let cursor = 0;
+        const covered: Unit[] = [];
+        for (const u of units.sort((a, b) => a.start - b.start)) {
+          if (covered.length) {
+            const whitespace =
+              source.slice(cursor, u.start).match(/^\s*/)?.[0].length ?? 0;
+            cursor += whitespace;
+            covered[covered.length - 1]!.end = cursor;
+          }
+          covered.push({ ...u, start: cursor });
+          cursor = u.end;
+        }
+        units = covered;
+        if (cursor < source.length) {
+          if (units.length) units[units.length - 1]!.end = source.length;
+          else units = [{ start: 0, end: source.length, kind: "declaration" }];
+        }
+      }
+    } finally {
+      tree?.delete();
+      parser.delete();
+    }
+  } else if (["markdown", "json", "yaml", "toml", "ini"].includes(lang)) {
+    status =
+      lang === "markdown" ? "markdown-boundaries" : "config-text-fallback";
+    units = textUnits(source, lang);
+  } else {
+    status = "unsupported-language-fallback";
+    units = [{ start: 0, end: source.length, kind: "fallback" }];
+  }
+  const fallback = mode === "window" || status.includes("fallback");
+  // Merge small neighboring non-function declarations only within the same scope/kind.
+  const merged: Unit[] = [];
+  for (const u of units) {
+    const prev = merged.at(-1);
+    if (
+      prev &&
+      prev.kind !== "function" &&
+      u.kind === prev.kind &&
+      u.scope === prev.scope &&
+      tokens(enrichment(path, prev) + source.slice(prev.start, u.end)) <=
+        CHUNKER.targetTokens
+    ) {
+      prev.end = u.end;
+      delete prev.symbol;
+      delete prev.signature;
+    } else merged.push({ ...u });
+  }
+  const bounded = merged.flatMap((u) =>
+    splitUnit(source, path, u, fallback ? CHUNKER.fallbackOverlap : 0),
+  );
+  const offsets = new Uint32Array(source.length + 1);
+  let byte = 0;
+  for (let i = 0; i < source.length;) {
+    const cp = source.codePointAt(i)!;
+    const chars = cp > 0xffff ? 2 : 1;
+    offsets[i] = byte;
+    if (chars === 2) offsets[i + 1] = byte;
+    byte += Buffer.byteLength(String.fromCodePoint(cp));
+    i += chars;
+    offsets[i] = byte;
+  }
+  const raw = Buffer.from(source);
+  const spans = bounded.map((u) => {
+    const startByte = offsets[u.start]!,
+      endByte = offsets[u.end]!;
+    const text = source.slice(u.start, u.end);
+    const searchText = enrichment(path, u) + text;
+    const kind = text.trim() === "" ? "structural" : u.kind;
+    return {
+      startByte,
+      endByte,
+      startLine: lineAt(raw, startByte),
+      endLine: lineAt(raw, Math.max(startByte, endByte - 1)),
+      chunkKind: kind,
+      scope: u.scope,
+      symbol: u.symbol,
+      signature: u.signature,
+      searchText,
+      tokenCount: tokens(searchText),
+    };
+  });
+  let covered = 0;
+  for (const s of spans) {
+    invariant(
+      s.startByte <= covered &&
+        s.endByte > s.startByte &&
+        s.tokenCount <= CHUNKER.maxTokens,
+      "Invalid chunk coverage or size.",
+    );
+    covered = Math.max(covered, s.endByte);
+  }
+  invariant(covered === raw.length, "Incomplete source coverage.");
+  return { spans, parseStatus: status, language: lang };
+}

package/src/cli.ts

@@ -0,0 +1,366 @@
+#!/usr/bin/env node
+import { readFileSync } from "node:fs";
+import { Command, InvalidArgumentError } from "commander";
+import { mkdir } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { randomUUID } from "node:crypto";
+import { configure, loadSettings, stateRoot } from "./settings.js";
+import { identity } from "./git.js";
+import { loadBuild, materialize, validateBuild, type Build } from "./build.js";
+import { invariant, optionalJson } from "./common.js";
+import { LambdaRemote } from "./remote.js";
+import {
+  attachment,
+  destination,
+  discover,
+  register,
+  selectRepository,
+} from "./repository.js";
+import { exclusive, publish, published, type Published } from "./publish.js";
+import { resolveVersion, syncTags } from "./releases.js";
+import { directHandle, loadHandle, readHandle, search } from "./search.js";
+const cli = new Command()
+  .name("srcx")
+  .description("Version-aware code search on LambdaDB")
+  .enablePositionalOptions()
+  .version(
+    JSON.parse(
+      readFileSync(new URL("../package.json", import.meta.url), "utf8"),
+    ).version,
+  );
+const output = (value: unknown): void => {
+  process.stdout.write(JSON.stringify(value, null, 2) + "\n");
+};
+const integer = (value: string) => {
+  if (!/^\d+$/.test(value))
+    throw new InvalidArgumentError("Expected a nonnegative integer.");
+  const n = Number(value);
+  if (!Number.isSafeInteger(n))
+    throw new InvalidArgumentError("Integer is too large.");
+  return n;
+};
+async function connected() {
+  const settings = await loadSettings();
+  return { settings, remote: new LambdaRemote(settings) };
+}
+cli
+  .command("configure")
+  .requiredOption("--endpoint <origin>")
+  .requiredOption("--project <name>")
+  .option(
+    "--api-key-env <name>",
+    "Environment variable containing the API key",
+    "LAMBDADB_API_KEY",
+  )
+  .action(async (o) =>
+    output(await configure(o.endpoint, o.project, o.apiKeyEnv)),
+  );
+cli
+  .command("doctor")
+  .description(
+    "Check authentication/read access; does not establish write/query readiness",
+  )
+  .action(async () => {
+    const { remote, settings } = await connected();
+    const collections = await remote.collections();
+    output({
+      endpoint: settings.endpoint,
+      project: settings.project,
+      readAccess: true,
+      collections: collections.length,
+      writeAccess: "not checked",
+      queryReadiness: "not checked",
+    });
+  });
+const repo = cli
+  .command("repo")
+  .description("Discover and attach Git repositories");
+repo
+  .command("add")
+  .requiredOption("--path <directory>")
+  .option("--remote <name>")
+  .option("--description <text>")
+  .option("--tag <key=value>", "One optional Collection metadata tag")
+  .action(async (o) => {
+    const { remote } = await connected();
+    let labels: Record<string, string> | undefined;
+    if (o.tag) {
+      const at = o.tag.indexOf("=");
+      invariant(at > 0, "Use --tag key=value.");
+      labels = { [o.tag.slice(0, at)]: o.tag.slice(at + 1) };
+    }
+    output(
+      await register(remote, {
+        path: o.path,
+        remote: o.remote,
+        description: o.description,
+        labels,
+      }),
+    );
+  });
+repo.command("list").action(async () => {
+  const { remote } = await connected();
+  output(await discover(remote));
+});
+repo
+  .command("show")
+  .requiredOption("--repo <name>")
+  .action(async (o) => {
+    const { remote } = await connected();
+    output(await selectRepository(remote, o.repo));
+  });
+cli
+  .command("import")
+  .description(
+    "Build a pinned Git commit; --dry-run exports locally without credentials",
+  )
+  .option(
+    "--path <directory>",
+    "Local checkout (required with credential-free --dry-run)",
+  )
+  .option("--remote <name>")
+  .option("--repo <name>", "Registered remote repository")
+  .option("--ref <ref>", "Local branch, Git tag, or commit OID")
+  .option("--dry-run", "Only materialize and validate a local build")
+  .option("--output <directory>", "New artifact directory")
+  .option(
+    "--previous <directory>",
+    "Previous build artifact for a dry-run diff",
+  )
+  .option("--artifact <directory>", "Import a previously previewed artifact")
+  .option(
+    "--resume",
+    "Resume the same pending artifact after inspecting its journal",
+  )
+  .option("--timeout <seconds>", "Validation wait deadline", integer, 120)
+  .action(async (o) => {
+    invariant(!(o.path && o.repo), "Choose --path or --repo.");
+    invariant(
+      !o.previous || o.dryRun,
+      "--previous is only accepted with --dry-run.",
+    );
+    invariant(!o.resume || !o.dryRun, "--resume requires a connected import.");
+    invariant(
+      !(o.artifact && (o.ref || o.path || o.output || o.previous)),
+      "--artifact cannot be combined with --ref/--path/--output/--previous.",
+    );
+    const buildLocal = async (
+      source: Awaited<ReturnType<typeof identity>>,
+      previous?: Build,
+    ) => {
+      invariant(o.ref, "--ref is required to build a commit.");
+      const parent = join(stateRoot(), "builds");
+      await mkdir(parent, { recursive: true, mode: 0o700 });
+      const path = o.output ? resolve(o.output) : join(parent, randomUUID());
+      return materialize({
+        identity: source,
+        ref: o.ref,
+        output: path,
+        previous,
+      });
+    };
+    if (o.dryRun) {
+      let b: Build;
+      if (o.artifact) {
+        b = await loadBuild(o.artifact);
+        await validateBuild(b);
+      } else {
+        let source;
+        if (o.path) source = await identity(o.path, o.remote);
+        else {
+          invariant(o.repo, "Use --path for a credential-free dry run.");
+          const { remote, settings } = await connected();
+          source = await attachment(
+            settings,
+            await selectRepository(remote, o.repo),
+          );
+        }
+        b = await buildLocal(
+          source,
+          o.previous ? await loadBuild(o.previous) : undefined,
+        );
+      }
+      output({
+        status: "locally-validated",
+        artifact: b.directory,
+        commitOid: b.commitOid,
+        configHash: b.configHash,
+        counts: b.counts,
+        changes: b.changes,
+        obsoleteIds: b.obsoleteIds,
+        coverage: b.inventory.map((e) => ({
+          path: e.path,
+          status: e.status,
+          reason: e.reason,
+          parseStatus: e.parseStatus,
+        })),
+        uploaded: false,
+      });
+      return;
+    }
+    invariant(
+      o.repo,
+      "Connected import requires --repo. Register it with repo add first.",
+    );
+    invariant(o.timeout > 0, "Timeout must be positive.");
+    const { remote, settings } = await connected();
+    const r = await selectRepository(remote, o.repo),
+      state = destination(settings, r.collection),
+      store = remote.store(r.collection);
+    output(
+      await exclusive(state, async () => {
+        // Pending attempts reconcile against the journal's immutable remote
+        // baseline; they do not need the previous build's local files.
+        const pending = await optionalJson(join(state, "pending.json"));
+        let baseline: { version: Published; build: Build } | undefined;
+        if (!pending) {
+          const last = await optionalJson<{
+            artifact: string;
+            version: Published;
+          }>(join(state, "last-build.json"));
+          if (last) {
+            const b = await loadBuild(last.artifact);
+            baseline = { version: last.version, build: b };
+          }
+        }
+        const b = o.artifact
+          ? await loadBuild(o.artifact)
+          : await buildLocal(await attachment(settings, r), baseline?.build);
+        return publish({
+          store,
+          binding: r,
+          build: b,
+          state,
+          baseline,
+          resume: o.resume,
+          timeoutMs: o.timeout * 1000,
+        });
+      }),
+    );
+  });
+cli
+  .command("versions")
+  .requiredOption("--repo <name>")
+  .action(async (o) => {
+    const { remote } = await connected();
+    const r = await selectRepository(remote, o.repo);
+    output(await published(remote.store(r.collection), r));
+  });
+cli
+  .command("resolve")
+  .requiredOption("--repo <name>")
+  .requiredOption("--ref <commit-or-release>")
+  .action(async (o) => {
+    const { remote } = await connected();
+    const r = await selectRepository(remote, o.repo);
+    output(await resolveVersion(remote.store(r.collection), r, o.ref));
+  });
+cli
+  .command("search")
+  .requiredOption("--repo <name>")
+  .requiredOption("--version <commit-or-release>")
+  .requiredOption("--query <text>")
+  .option("--limit <count>", "Maximum hits", integer, 10)
+  .option("--path <path>", "Exact path filter")
+  .option("--language <name>")
+  .action(async (o) => {
+    const { remote, settings } = await connected();
+    const r = await selectRepository(remote, o.repo),
+      store = remote.store(r.collection),
+      v = await resolveVersion(store, r, o.version);
+    output(
+      await search(store, settings, r, v, o.query, o.limit, {
+        path: o.path,
+        language: o.language,
+      }),
+    );
+  });
+cli
+  .command("read")
+  .option("--result <id>")
+  .option("--repo <name>")
+  .option("--version <commit-or-release>")
+  .option("--path <path>")
+  .option("--lines <start:end>")
+  .option("--context <lines>", "Expand an evidence span", integer, 0)
+  .option("--full-file")
+  .action(async (o) => {
+    invariant(
+      !(o.fullFile && (o.lines || o.context)),
+      "--full-file cannot be combined with --lines or --context.",
+    );
+    invariant(
+      !(o.result && (o.repo || o.version || o.path || o.lines)),
+      "Use result mode or direct path mode.",
+    );
+    const { settings, remote } = await connected();
+    let handle;
+    if (o.result) {
+      handle = await loadHandle(o.result);
+      invariant(
+        handle.endpoint === settings.endpoint &&
+          handle.project === settings.project,
+        "Result belongs to another endpoint/project.",
+      );
+    } else {
+      invariant(
+        o.repo && o.version && o.path,
+        "Provide --result, or --repo/--version/--path.",
+      );
+      const r = await selectRepository(remote, o.repo),
+        store = remote.store(r.collection);
+      handle = await directHandle(
+        store,
+        settings,
+        r,
+        await resolveVersion(store, r, o.version),
+        o.path,
+      );
+    }
+    let lines: [number, number] | undefined;
+    if (o.lines) {
+      invariant(/^\d+:\d+$/.test(o.lines), "Use --lines start:end.");
+      lines = o.lines.split(":").map(Number) as [number, number];
+    }
+    output(
+      await readHandle(
+        remote.store(handle.repository.collection),
+        settings,
+        handle,
+        {
+          context: o.context,
+          fullFile: o.fullFile || (!o.result && !o.lines),
+          lines,
+        },
+      ),
+    );
+  });
+cli
+  .command("git")
+  .command("sync-tags")
+  .description(
+    "Synchronize currently observed local Git tags; no fetch/import/pruning",
+  )
+  .requiredOption("--repo <name>")
+  .action(async (o) => {
+    const { remote, settings } = await connected();
+    const r = await selectRepository(remote, o.repo),
+      source = await attachment(settings, r),
+      state = destination(settings, r.collection);
+    output(
+      await exclusive(state, async () => {
+        invariant(
+          !(await optionalJson(join(state, "pending.json"))),
+          "An import is pending; resume it before tag synchronization.",
+        );
+        return syncTags(remote.store(r.collection), r, source.path);
+      }),
+    );
+  });
+try {
+  await cli.parseAsync();
+} catch (e) {
+  const message = e instanceof Error ? e.message : "Operation failed.";
+  process.stderr.write(`srcx: ${message}\n`);
+  process.exitCode = 1;
+}