@hraness/wordcell 0.20.0-bootstrap.1 → 0.21.0

package/src/graph-cli.ts

@@ -0,0 +1,96 @@
+import { queryGraph, rebuildGraph, verifyGraph } from "./graph-authority.js";
+import type { GraphQueryRequest, GraphQueryResult, GraphVerification } from "./graph-authority-model.js";
+import { validateGraphQueryRequest } from "./graph-query.js";
+
+type GraphCliBase = Readonly<{ root: string; index?: string; json: boolean }>;
+export type GraphCliCommand =
+  | GraphCliBase & Readonly<{ kind: "graph-rebuild"; fresh: boolean }>
+  | GraphCliBase & Readonly<{ kind: "graph-verify" }>
+  | GraphCliBase & Readonly<{ kind: "graph-query"; request: GraphQueryRequest; persisted: boolean }>;
+export type GraphCliParseResult =
+  | Readonly<{ ok: true; value: GraphCliCommand }>
+  | Readonly<{ ok: false; message: string }>;
+export type GraphCliDependencies = Readonly<{
+  queryGraph?: typeof queryGraph;
+  rebuildGraph?: typeof rebuildGraph;
+  verifyGraph?: typeof verifyGraph;
+}>;
+
+export function parseGraphCommand(arguments_: readonly string[]): GraphCliParseResult {
+  const action = arguments_[0];
+  if (action !== "rebuild" && action !== "verify" && action !== "query") {
+    return { ok: false, message: "unknown graph action" };
+  }
+  if (arguments_.length > 32) return { ok: false, message: "too many graph options" };
+  let root = ".";
+  let index: string | undefined;
+  let json = false;
+  let persisted = false;
+  let fresh = false;
+  const request: Record<string, unknown> = {};
+  const seen = new Set<string>();
+  for (let position = 1; position < arguments_.length; position += 1) {
+    const option = arguments_[position]!;
+    if (seen.has(option)) return { ok: false, message: "duplicate graph option" };
+    seen.add(option);
+    if (option === "--json") { json = true; continue; }
+    if (option === "--fresh" && action === "rebuild") { fresh = true; continue; }
+    if (option === "--persisted" && action === "query") { persisted = true; continue; }
+    if (option !== "--root" && option !== "--index"
+      && !(action === "query" && ["--program", "--note", "--scope", "--predicate", "--depth", "--limit"].includes(option))) {
+      return { ok: false, message: "unknown graph option" };
+    }
+    const value = arguments_[++position];
+    if (value === undefined || value.startsWith("--") || value.length === 0 || Buffer.byteLength(value, "utf8") > 16_384) {
+      return { ok: false, message: "graph option requires a bounded value" };
+    }
+    if (option === "--root") { root = value; continue; }
+    if (option === "--index") { index = value; continue; }
+    if (option === "--depth" || option === "--limit") {
+      if (!/^[1-9][0-9]*$/u.test(value) || !Number.isSafeInteger(Number(value))) {
+        return { ok: false, message: "graph count must be a positive integer" };
+      }
+      if (option === "--depth") request.depth = Number(value);
+      else request.limits = { rows: Number(value) };
+    } else request[option.slice(2)] = value;
+  }
+  const base = { root, ...(index === undefined ? {} : { index }), json };
+  if (action === "rebuild") return { ok: true, value: { kind: "graph-rebuild", ...base, fresh } };
+  if (action === "verify") return { ok: true, value: { kind: "graph-verify", ...base } };
+  try {
+    return { ok: true, value: { kind: "graph-query", ...base, persisted, request: validateGraphQueryRequest(request) } };
+  } catch (error) {
+    return { ok: false, message: error instanceof Error ? error.message : "invalid graph query" };
+  }
+}
+
+export async function executeGraphCommand(
+  command: GraphCliCommand,
+  dependencies: GraphCliDependencies = {},
+): Promise<Readonly<{ value: GraphQueryResult | GraphVerification; text: string; exitCode: number }>> {
+  const options = command.index === undefined ? {} : { index: command.index };
+  if (command.kind === "graph-query") {
+    const value = await (dependencies.queryGraph ?? queryGraph)(command.root, command.request, {
+      ...options, persisted: command.persisted,
+    });
+    const lines = [
+      `${value.request.program}: ${value.rows.length} derived graph rows.`,
+      `Snapshot: ${value.revision}`,
+      "Proofs and exact source record digests are included in --json output.",
+      value.columns.join("\t"),
+      ...value.rows.map((row) => row.values.map((atom) => JSON.stringify(atom)).join("\t")),
+    ];
+    if (value.truncated || value.proofsTruncated) {
+      lines.push(`Partial result: ${value.truncationReasons.join(", ") || "proof evidence was truncated"}.`);
+    }
+    return { value, text: `${lines.join("\n")}\n`, exitCode: value.truncated || value.proofsTruncated ? 4 : 0 };
+  }
+  const value = command.kind === "graph-rebuild"
+    ? await (dependencies.rebuildGraph ?? rebuildGraph)(command.root, { ...options, fresh: command.fresh })
+    : await (dependencies.verifyGraph ?? verifyGraph)(command.root, options);
+  return {
+    value,
+    text: `${command.kind === "graph-rebuild" ? "Rebuilt" : "Verified"} the local graph projection: ${value.records} notes, ${value.facts} facts.\nSnapshot: ${value.revision}\n`,
+    exitCode: 0,
+  };
+}

package/src/graph-facts.ts

@@ -0,0 +1,206 @@
+import { createHash } from "node:crypto";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+
+import {
+  analyzeVault,
+  isCanonicalNoteId,
+  parseNote,
+  VaultAnalysisBudgetError,
+  type Note,
+} from "./graph.js";
+import {
+  GRAPH_LIMITS,
+  GraphAuthorityError,
+  type GraphAtom,
+  type GraphFact,
+  type GraphFactRelation,
+  type GraphSnapshot,
+  type GraphSourceRecord,
+} from "./graph-authority-model.js";
+import { documentIdState } from "./portfolio-identity.js";
+import { analyzeAuthoredRepositoryScopes } from "./repository-memory.js";
+import type { VaultSnapshot } from "./vault.js";
+
+const compareText = (left: string, right: string): number => left < right ? -1 : left > right ? 1 : 0;
+const sha256 = (value: string): string => createHash("sha256").update(value, "utf8").digest("hex");
+
+/** Internal protocol serialization: only freshly constructed, finite JSON values enter here. */
+function canonicalJson(value: unknown): string {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
+  return `{${Object.entries(value).toSorted(([left], [right]) => compareText(left, right))
+    .map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`;
+}
+
+function invalid(message: string): never {
+  throw new GraphAuthorityError("invalid-input", message);
+}
+
+function budget(message: string): never {
+  throw new GraphAuthorityError("budget", message);
+}
+
+function textAtom(value: unknown, label: string): string {
+  if (typeof value !== "string") invalid(`${label} must be a string.`);
+  if (Buffer.byteLength(value, "utf8") > GRAPH_LIMITS.atomBytes) {
+    budget(`${label} exceeds the ${GRAPH_LIMITS.atomBytes} byte graph atom limit.`);
+  }
+  return value;
+}
+
+function markdownPath(value: unknown, label: string): string {
+  const path = textAtom(value, label);
+  if (!path.toLowerCase().endsWith(".md") || !isCanonicalNoteId(path.slice(0, -3))) {
+    invalid(`${label} must be a canonical vault-relative Markdown path.`);
+  }
+  return path;
+}
+
+type CheckedSource = Readonly<{ note: Note; documentId: string | null; scopes: readonly string[] }>;
+
+/**
+ * Derive a bounded, immutable graph edition from authored Markdown.
+ * Caller-provided parsed fields and analysis are not evidence: facts are reparsed
+ * from the same bytes whose digests bind every returned source record.
+ */
+export function createGraphSnapshot(snapshot: VaultSnapshot): GraphSnapshot {
+  if (snapshot === null || typeof snapshot !== "object") invalid("A vault snapshot is required.");
+  const rootInput = textAtom(snapshot.root, "Vault root");
+  if (!isAbsolute(rootInput) || rootInput.includes("\0")) invalid("Vault root must be absolute.");
+  const root = resolve(rootInput);
+  const indexPath = textAtom(snapshot.indexPath, "Catalog path");
+  if (!isAbsolute(indexPath) || indexPath.includes("\0")) invalid("Catalog path must be absolute.");
+  const catalogPath = markdownPath(relative(root, resolve(indexPath)).split(sep).join("/"), "Catalog path");
+  const catalogNoteId = catalogPath.slice(0, -3);
+  if (!Array.isArray(snapshot.notes)) invalid("Vault notes must be an array.");
+  if (snapshot.notes.length > GRAPH_LIMITS.notes) {
+    budget(`Graph exceeds the ${GRAPH_LIMITS.notes} note limit.`);
+  }
+
+  let sourceBytes = 0;
+  const paths = new Set<string>();
+  // Bound all source bytes and identities before parsing or materializing facts.
+  const sources = snapshot.notes.map((value: unknown) => {
+    if (value === null || typeof value !== "object") invalid("Every vault note must be an object.");
+    const candidate = value as Record<string, unknown>;
+    const path = markdownPath(candidate.path, "Note path");
+    if (candidate.id !== path.slice(0, -3)) invalid(`Note ID must match its canonical path: ${path}.`);
+    if (paths.has(path)) invalid(`Duplicate note path: ${path}.`);
+    paths.add(path);
+    if (typeof candidate.content !== "string") invalid(`Note content must be a string: ${path}.`);
+    sourceBytes += Buffer.byteLength(candidate.content, "utf8");
+    if (sourceBytes > GRAPH_LIMITS.sourceBytes) {
+      budget(`Graph sources exceed the ${GRAPH_LIMITS.sourceBytes} byte limit.`);
+    }
+    return { path, content: candidate.content };
+  }).toSorted((left, right) => compareText(left.path, right.path));
+
+  const documentIds = new Set<string>();
+  const noteIds = new Set<string>();
+  let connectionObservations = 0;
+  const checked: CheckedSource[] = sources.map(({ path, content }) => {
+    let note: Note;
+    try {
+      note = parseNote(path, content);
+    } catch {
+      invalid(`Cannot parse authored Markdown: ${path}.`);
+    }
+    if (noteIds.has(note.id)) invalid(`Duplicate note ID: ${note.id}.`);
+    noteIds.add(note.id);
+    textAtom(note.title, `Title in ${path}`);
+    textAtom(note.properties.type ?? "", `Type in ${path}`);
+    for (const tag of note.tags) textAtom(tag, `Tag in ${path}`);
+    const identity = documentIdState(note.metadata);
+    if (identity.kind === "invalid") invalid(`Invalid document_id in ${path}.`);
+    const documentId = identity.kind === "valid" ? identity.documentId : null;
+    if (documentId !== null) {
+      if (documentIds.has(documentId)) invalid(`Duplicate document_id: ${documentId}.`);
+      documentIds.add(documentId);
+    }
+    const scopes = analyzeAuthoredRepositoryScopes(note.metadata);
+    if (!scopes.valid) invalid(`Invalid repository_scopes in ${path}.`);
+    for (const scope of scopes.scopes) textAtom(scope, `Repository scope in ${path}`);
+    connectionObservations += note.links.length
+      + (note.relationDeclarations?.length ?? 0) + (note.relationIssues?.length ?? 0);
+    if (connectionObservations > GRAPH_LIMITS.facts) {
+      budget(`Graph connections exceed the ${GRAPH_LIMITS.facts} observation limit.`);
+    }
+    return { note, documentId, scopes: scopes.scopes };
+  });
+
+  let analysis: ReturnType<typeof analyzeVault>;
+  try {
+    analysis = analyzeVault(checked.map(({ note }) => note), {
+      catalogNoteId,
+      mentionScope: () => false,
+      maxNotes: GRAPH_LIMITS.notes,
+      maxConnectionObservations: GRAPH_LIMITS.facts,
+    });
+  } catch (error) {
+    if (error instanceof VaultAnalysisBudgetError) budget(error.message);
+    throw error;
+  }
+
+  const included = new Set(analysis.noteConnections.map(({ id }) => id));
+  const factGroups = new Map<string, Map<string, GraphFact>>();
+  let factCount = 0;
+  const add = (source: string, relation: GraphFactRelation, tuple: readonly GraphAtom[]): void => {
+    if (!included.has(source)) invalid(`Graph fact has no authored source: ${source}.`);
+    for (const atom of tuple) {
+      if (typeof atom === "string") textAtom(atom, relation);
+      else if (typeof atom === "number" && (!Number.isSafeInteger(atom) || atom < 1)) {
+        invalid(`Graph source lines must be positive safe integers: ${source}.`);
+      }
+    }
+    const key = canonicalJson({ relation, tuple });
+    const group = factGroups.get(source) ?? new Map<string, GraphFact>();
+    if (group.has(key)) return;
+    if (factCount >= GRAPH_LIMITS.facts) budget(`Graph exceeds the ${GRAPH_LIMITS.facts} fact limit.`);
+    factCount += 1;
+    group.set(key, Object.freeze({ relation, tuple: Object.freeze([...tuple]) }));
+    factGroups.set(source, group);
+  };
+  for (const { note, scopes } of checked) {
+    if (!included.has(note.id)) continue;
+    const type = note.properties.type ?? "";
+    add(note.id, "wordcell.note", [note.id, note.path, note.title, type]);
+    if (type.normalize("NFC").toLocaleLowerCase("en-US") === "concept") {
+      add(note.id, "wordcell.concept", [note.id]);
+    }
+    for (const tag of note.tags) add(note.id, "wordcell.tag", [note.id, tag]);
+    for (const scope of scopes) add(note.id, "wordcell.scope", [note.id, scope]);
+  }
+  for (const link of analysis.contextualLinks) {
+    const source = link.source.slice(0, -3);
+    add(source, "wordcell.link", [source, link.target.slice(0, -3), link.line]);
+  }
+  for (const relation of analysis.authoredRelations) {
+    add(relation.source, "wordcell.relation", [
+      relation.source, relation.target, relation.predicate, relation.provenance.line,
+    ]);
+  }
+  for (const relation of analysis.externalAuthoredRelations) {
+    add(relation.source, "wordcell.external-relation", [
+      relation.source, relation.target, relation.predicate, relation.provenance.line,
+    ]);
+  }
+
+  const sourceDigests = Object.freeze(sources.map(({ path, content }) =>
+    Object.freeze({ path, contentSha256: sha256(content) })));
+  const digests = new Map(sourceDigests.map(({ path, contentSha256 }) => [path, contentSha256]));
+  const records: readonly GraphSourceRecord[] = Object.freeze(checked
+    .filter(({ note }) => included.has(note.id))
+    .map(({ note, documentId }): GraphSourceRecord => Object.freeze({
+      key: `edition:note/${sha256(documentId === null ? `path:${note.id}` : `document:${documentId}`)}`,
+      id: note.id,
+      path: note.path,
+      documentId,
+      contentSha256: digests.get(note.path)!,
+      facts: Object.freeze([...(factGroups.get(note.id)?.entries() ?? [])]
+        .toSorted(([left], [right]) => compareText(left, right)).map(([, fact]) => fact)),
+    }))
+    .toSorted((left, right) => compareText(left.key, right.key)));
+  const identity = Object.freeze({ schemaVersion: 1 as const, vaultIdentity: sha256(root),
+    catalogNoteId, sourceDigests, records });
+  return Object.freeze({ ...identity, revision: sha256(canonicalJson(identity)), factCount });
+}

package/src/graph-percolation.ts

@@ -0,0 +1,39 @@
+import { analyzeVault, lookupNote, parseNote } from "./graph.js";
+import { createGraphSnapshot, openGraphAuthority, GraphAuthorityError } from "./graph-authority.js";
+import type { GraphQueryResult } from "./graph-authority.js";
+import { validateGraphPercolationOptions } from "./graph-query.js";
+import { percolateVault } from "./percolate.js";
+import type { PercolateOptions, PercolationResultV2 } from "./percolate.js";
+import type { VaultSnapshot } from "./vault.js";
+
+export type GraphPercolationResult = Readonly<{
+  schemaVersion: 1;
+  kind: "wordcell.graph-percolation";
+  revision: string;
+  suggestions: PercolationResultV2;
+  /** Positive shared-tag/concept support only. Absence, counts and predicate decisions remain host-owned. */
+  positiveSupport: Readonly<{ sharedTags: GraphQueryResult; sharedConcepts: GraphQueryResult }>;
+}>;
+
+/** Add bounded, separately inspectable positive proofs without changing percolation V2 candidates. */
+export async function percolateWithGraph(snapshot: VaultSnapshot,
+  options: PercolateOptions & Readonly<{ note: string }>): Promise<GraphPercolationResult> {
+  options = validateGraphPercolationOptions(options);
+  const graph = createGraphSnapshot(snapshot);
+  const notes = snapshot.notes.map(note => parseNote(note.path, note.content));
+  const selected = lookupNote(notes, options.note);
+  if (selected.kind !== "found" || !graph.records.some(record => record.id === selected.note.id)) {
+    throw new GraphAuthorityError("invalid-input", "Proof-backed percolation requires one unambiguous content note.");
+  }
+  const analysis = analyzeVault(notes, { catalogNoteId: graph.catalogNoteId,
+    mentionScope: note => note.id === selected.note.id, maxMentionPairs: 250_000 });
+  const suggestions = percolateVault(notes, analysis, options);
+  const authority = await openGraphAuthority(snapshot);
+  try {
+    const limits = { rows: 1_000, workUnits: 1_000_000, resultBytes: 1024 * 1024 };
+    const sharedTags = await authority.query({ program: "shared-tags", note: selected.note.id, limits });
+    const sharedConcepts = await authority.query({ program: "shared-concepts", note: selected.note.id, limits });
+    return Object.freeze({ schemaVersion: 1, kind: "wordcell.graph-percolation", revision: graph.revision,
+      suggestions, positiveSupport: Object.freeze({ sharedTags, sharedConcepts }) });
+  } finally { await authority.close(); }
+}

package/src/graph-query.ts

@@ -0,0 +1,122 @@
+import { isProxy } from "node:util/types";
+import { isCanonicalNoteId, isCanonicalRelationPredicate } from "./graph.js";
+import {
+  GRAPH_LIMITS,
+  GraphAuthorityError,
+  type GraphQueryLimits,
+  type GraphQueryRequest,
+} from "./graph-authority-model.js";
+import type { PercolateOptions } from "./percolate.js";
+import { validateRepositoryScopeSelection } from "./repository-memory.js";
+
+function invalid(message: string): never {
+  throw new GraphAuthorityError("invalid-input", message);
+}
+
+function dataObject(value: unknown, label: string): Readonly<Record<string, unknown>> {
+  if (value === null || typeof value !== "object" || isProxy(value) || Array.isArray(value)) {
+    return invalid(`${label} must be a plain data object.`);
+  }
+  const prototype: unknown = Object.getPrototypeOf(value);
+  if (prototype !== Object.prototype && prototype !== null) {
+    return invalid(`${label} must be a plain data object.`);
+  }
+  const keys = Reflect.ownKeys(value);
+  if (keys.length > 16) return invalid(`${label} contains too many fields.`);
+  const result: Record<string, unknown> = Object.create(null) as Record<string, unknown>;
+  for (const key of keys) {
+    if (typeof key !== "string") return invalid(`${label} must have only named data fields.`);
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    if (descriptor === undefined || !("value" in descriptor) || !descriptor.enumerable) {
+      return invalid(`${label} must have only enumerable data fields.`);
+    }
+    result[key] = descriptor.value;
+  }
+  return result;
+}
+
+function boundedText(value: unknown, label: string): string {
+  if (typeof value !== "string" || Buffer.byteLength(value, "utf8") > GRAPH_LIMITS.atomBytes) {
+    return invalid(`${label} must be a bounded string.`);
+  }
+  return value;
+}
+
+function boundedInteger(value: unknown, maximum: number, label: string): number {
+  if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1 || value > maximum) {
+    return invalid(`${label} must be an integer from 1 through ${maximum}.`);
+  }
+  return value;
+}
+
+function checkedLimits(value: unknown): GraphQueryLimits {
+  const input = dataObject(value, "Graph query limits");
+  const result: Record<string, number> = {};
+  const keys = ["rows", "workUnits", "derivedTuples", "rounds", "proofDepth", "proofNodes",
+    "totalProofNodes", "resultBytes"] as const;
+  for (const key of Object.keys(input)) {
+    if (!(keys as readonly string[]).includes(key)) return invalid("Unknown graph query limit.");
+  }
+  for (const key of keys) {
+    if (Object.hasOwn(input, key)) {
+      result[key] = boundedInteger(input[key], GRAPH_LIMITS[key], `Graph query ${key}`);
+    }
+  }
+  return Object.freeze(result);
+}
+
+/** Validate and copy a named query before any scan, store open, or asynchronous work. */
+export function validateGraphQueryRequest(value: unknown): GraphQueryRequest {
+  const input = dataObject(value, "Graph query");
+  const program = input.program;
+  if (program !== "backlinks" && program !== "reachability" && program !== "scope-route"
+    && program !== "relation-closure" && program !== "shared-tags" && program !== "shared-concepts") {
+    return invalid("Graph query requires a supported named program.");
+  }
+  const allowed = new Set(["program", "limits", program === "scope-route" ? "scope" : "note"]);
+  if (program === "reachability" || program === "relation-closure") allowed.add("depth");
+  if (program === "relation-closure") allowed.add("predicate");
+  if (Object.keys(input).some((key) => !allowed.has(key))) {
+    return invalid("Graph query contains an option unsupported by its named program.");
+  }
+  const limits = Object.hasOwn(input, "limits") ? checkedLimits(input.limits) : undefined;
+  const extras = limits === undefined ? {} : { limits };
+  if (program === "scope-route") {
+    const scope = boundedText(input.scope, "Graph query scope");
+    try { validateRepositoryScopeSelection([scope]); } catch {
+      return invalid("Graph query scope must be an exact canonical repository path.");
+    }
+    return Object.freeze({ program, scope, ...extras });
+  }
+  const note = boundedText(input.note, "Graph query note");
+  if (!isCanonicalNoteId(note)) return invalid("Graph query note must be an exact canonical note ID.");
+  const depth = Object.hasOwn(input, "depth")
+    ? boundedInteger(input.depth, GRAPH_LIMITS.depth, "Graph query depth") : undefined;
+  const depthOption = depth === undefined ? {} : { depth };
+  if (program === "relation-closure") {
+    const predicate = boundedText(input.predicate, "Graph query predicate");
+    if (!isCanonicalRelationPredicate(predicate)) {
+      return invalid("Graph query predicate must be an exact lower-kebab relation predicate.");
+    }
+    return Object.freeze({ program, note, predicate, ...depthOption, ...extras });
+  }
+  if (program === "reachability") return Object.freeze({ program, note, ...depthOption, ...extras });
+  return Object.freeze({ program, note, ...extras });
+}
+
+/** Copy bounded advisory options without evaluating foreign getters. Note lookup remains Wordcell-owned. */
+export function validateGraphPercolationOptions(value: unknown): PercolateOptions & Readonly<{ note: string }> {
+  const input = dataObject(value, "Graph percolation options");
+  if (Object.keys(input).some((key) => !["note", "minSupport", "limit"].includes(key))) {
+    return invalid("Unknown graph percolation option.");
+  }
+  const note = boundedText(input.note, "Graph percolation note");
+  if (note.trim() === "") return invalid("Graph percolation requires one note.");
+  const minSupport = Object.hasOwn(input, "minSupport")
+    ? boundedInteger(input.minSupport, 1_000, "Graph percolation minSupport") : undefined;
+  if (minSupport === 1) return invalid("Graph percolation minSupport must be at least 2.");
+  const limit = Object.hasOwn(input, "limit")
+    ? boundedInteger(input.limit, 1_000, "Graph percolation limit") : undefined;
+  return Object.freeze({ note, ...(minSupport === undefined ? {} : { minSupport }),
+    ...(limit === undefined ? {} : { limit }) });
+}

package/src/index.ts

@@ -19,3 +19,5 @@ export * from "./sdk.js";
 export * from "./source-inbox.js";
 export * from "./vault.js";
 export * from "./workflow.js";
+export * from "./graph-authority.js";
+export * from "./graph-percolation.js";