@danypops/lector 0.1.9 → 0.1.10

package/README.md

@@ -64,7 +64,7 @@ Three independent `SymbolIndexPort` implementations use one result DTO with expl
 - **TypeScript compiler** extracts bounded structural declarations from cold, malformed, or partially configured projects. It does not claim cross-file identity.
 - **tree-sitter** extracts bounded structural declarations and caches them by content hash. It does not claim type or language-server identity.
 
-File-position operations dispatch by extension and seed a cold server with the requested file. Workspace symbol search and graph population detect every enabled language, merge results deterministically, and retain per-symbol and per-backend provenance; one failed backend produces an explicit partial result instead of hiding successful languages. Bash remains explicit-file-only until its vulnerable upstream dependency chain is removed; generic YAML is explicit-file-only because lockfiles and data files do not establish a workspace language.
+File-position operations dispatch by extension and seed a cold server with the requested file. Workspace symbol search and graph population detect every enabled language, merge results deterministically, and retain per-symbol and per-backend provenance. Backend and file-level semantic failures produce bounded partial results without discarding successful languages or files. Bash remains explicit-file-only until its vulnerable upstream dependency chain is removed; generic YAML is explicit-file-only because lockfiles and data files do not establish a workspace language.
 
 Symbol results report `fidelity`, `backend`, `authority`, `freshness`, `limitations`, `truncated`, and composite `sources`/`completeness` when applicable. Language-server messages, pending requests, opened files, file bytes, settling, parser files, parser nodes, parser bytes, language indexes, and returned symbols are bounded.
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@danypops/lector",
-	"version": "0.1.9",
+	"version": "0.1.10",
 	"description": "Platform-neutral filesystem & code-intelligence capability: core, service, and driven adapters",
 	"license": "MIT",
 	"type": "module",

package/src/adapters/polyglot-code-intelligence-index.ts

@@ -97,6 +97,10 @@ export class PolyglotCodeIntelligenceIndex implements SymbolIndexPort, CodeIntel
 		};
 	}
 
+	provenanceForPath(path: string): IntelligenceProvenance {
+		return this.indexForPath(path).provenance;
+	}
+
 	goToDefinition(at: WorkspaceLocation) {
 		return this.indexForPath(at.path).goToDefinition(at);
 	}

package/src/adapters/sqlite-symbol-graph.ts

@@ -1,6 +1,7 @@
 import type { Database } from "bun:sqlite";
 import { type Migration, openSqliteWithPragmas } from "@danypops/daemon-kit/storage";
 import type { IntelligenceProvenance } from "../domain/intelligence-provenance.ts";
+import type { PopulateSymbolGraphResult, SymbolGraphPopulationFailure } from "../domain/populate-symbol-graph.ts";
 import type { SymbolGraphGeneration } from "../domain/symbol-graph-generation.ts";
 import type { SymbolNodeId } from "../domain/symbol-node-id.ts";
 import type { SymbolEdgeKind, SymbolGraphPort, SymbolNode } from "../ports/symbol-graph-port.ts";
@@ -32,6 +33,10 @@ const MIGRATIONS: Migration[] = [
 		version: 4,
 		up: (db) => db.exec("ALTER TABLE symbol_graph_generation ADD COLUMN sources_json TEXT"),
 	},
+	{
+		version: 5,
+		up: (db) => db.exec("ALTER TABLE symbol_graph_generation ADD COLUMN result_json TEXT"),
+	},
 ];
 
 interface NodeRow {
@@ -53,6 +58,7 @@ interface GenerationRow {
 	edges_added: number;
 	provenance_json: string | null;
 	sources_json: string | null;
+	result_json: string | null;
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {
@@ -103,6 +109,69 @@ function parseSources(json: string | null): readonly IntelligenceProvenance[] |
 	return sources.every((source) => source !== undefined) ? sources : undefined;
 }
 
+function isNonNegativeInteger(value: unknown): value is number {
+	return Number.isSafeInteger(value) && typeof value === "number" && value >= 0;
+}
+
+function asPopulationFailure(value: unknown): SymbolGraphPopulationFailure | undefined {
+	if (!isRecord(value)) return undefined;
+	const provenance = asProvenance(value.provenance);
+	if (
+		typeof value.path !== "string" ||
+		value.path.length > 4096 ||
+		(value.operation !== "document-symbols" && value.operation !== "outgoing-calls") ||
+		typeof value.code !== "string" ||
+		value.code.length > 100 ||
+		typeof value.message !== "string" ||
+		value.message.length > 500 ||
+		!provenance
+	) {
+		return undefined;
+	}
+	return { path: value.path, operation: value.operation, code: value.code, message: value.message, provenance };
+}
+
+function parsePopulationResult(json: string | null): PopulateSymbolGraphResult | undefined {
+	const value = parseJson(json);
+	if (!isRecord(value) || (value.completeness !== "complete" && value.completeness !== "partial") || !Array.isArray(value.failures)) return undefined;
+	if (
+		!isNonNegativeInteger(value.filesAttempted) ||
+		!isNonNegativeInteger(value.filesProcessed) ||
+		!isNonNegativeInteger(value.filesFailed) ||
+		!isNonNegativeInteger(value.symbolsProcessed) ||
+		!isNonNegativeInteger(value.nodesAdded) ||
+		!isNonNegativeInteger(value.edgesAdded) ||
+		!isNonNegativeInteger(value.failureCount) ||
+		typeof value.failuresTruncated !== "boolean" ||
+		value.failures.length > 100
+	) {
+		return undefined;
+	}
+	const failures = value.failures.map(asPopulationFailure);
+	if (
+		!failures.every((failure) => failure !== undefined) ||
+		value.filesProcessed > value.filesAttempted ||
+		value.filesFailed > value.filesAttempted ||
+		failures.length > value.failureCount ||
+		(value.completeness === "complete") !== (value.failureCount === 0) ||
+		value.failuresTruncated !== value.failureCount > failures.length
+	) {
+		return undefined;
+	}
+	return {
+		completeness: value.completeness,
+		filesAttempted: value.filesAttempted,
+		filesProcessed: value.filesProcessed,
+		filesFailed: value.filesFailed,
+		symbolsProcessed: value.symbolsProcessed,
+		nodesAdded: value.nodesAdded,
+		edgesAdded: value.edgesAdded,
+		failureCount: value.failureCount,
+		failures,
+		failuresTruncated: value.failuresTruncated,
+	};
+}
+
 /**
  * SQLite-backed SymbolGraphPort; survives a daemon restart pointed at the
  * same database file. reachableFrom uses `WITH RECURSIVE` rather than an
@@ -172,12 +241,26 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
 	async getGeneration(): Promise<SymbolGraphGeneration | undefined> {
 		const row = this.db
 			.query(
-				"SELECT source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json, sources_json FROM symbol_graph_generation WHERE singleton = 1",
+				"SELECT source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json, sources_json, result_json FROM symbol_graph_generation WHERE singleton = 1",
 			)
 			.get() as GenerationRow | null;
 		if (!row) return undefined;
 		const provenance = parseProvenance(row.provenance_json);
 		const sources = parseSources(row.sources_json);
+		const result =
+			parsePopulationResult(row.result_json) ??
+			({
+				completeness: "complete",
+				filesAttempted: row.files_processed,
+				filesProcessed: row.files_processed,
+				filesFailed: 0,
+				symbolsProcessed: row.symbols_processed,
+				nodesAdded: row.nodes_added,
+				edgesAdded: row.edges_added,
+				failureCount: 0,
+				failures: [],
+				failuresTruncated: false,
+			} satisfies PopulateSymbolGraphResult);
 		return {
 			sourceFingerprint: row.source_fingerprint,
 			maxFiles: row.max_files,
@@ -185,19 +268,14 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
 			completedAt: row.completed_at,
 			...(provenance ? { provenance } : {}),
 			...(sources ? { sources } : {}),
-			result: {
-				filesProcessed: row.files_processed,
-				symbolsProcessed: row.symbols_processed,
-				nodesAdded: row.nodes_added,
-				edgesAdded: row.edges_added,
-			},
+			result,
 		};
 	}
 
 	async setGeneration(generation: SymbolGraphGeneration): Promise<void> {
 		this.db
 			.query(
-				"INSERT INTO symbol_graph_generation (singleton, source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json, sources_json) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(singleton) DO UPDATE SET source_fingerprint = excluded.source_fingerprint, max_files = excluded.max_files, max_symbols_per_file = excluded.max_symbols_per_file, completed_at = excluded.completed_at, files_processed = excluded.files_processed, symbols_processed = excluded.symbols_processed, nodes_added = excluded.nodes_added, edges_added = excluded.edges_added, provenance_json = excluded.provenance_json, sources_json = excluded.sources_json",
+				"INSERT INTO symbol_graph_generation (singleton, source_fingerprint, max_files, max_symbols_per_file, completed_at, files_processed, symbols_processed, nodes_added, edges_added, provenance_json, sources_json, result_json) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(singleton) DO UPDATE SET source_fingerprint = excluded.source_fingerprint, max_files = excluded.max_files, max_symbols_per_file = excluded.max_symbols_per_file, completed_at = excluded.completed_at, files_processed = excluded.files_processed, symbols_processed = excluded.symbols_processed, nodes_added = excluded.nodes_added, edges_added = excluded.edges_added, provenance_json = excluded.provenance_json, sources_json = excluded.sources_json, result_json = excluded.result_json",
 			)
 			.run(
 				generation.sourceFingerprint,
@@ -210,6 +288,7 @@ export class SqliteSymbolGraph implements SymbolGraphPort {
 				generation.result.edgesAdded,
 				generation.provenance ? JSON.stringify(generation.provenance) : null,
 				generation.sources ? JSON.stringify(generation.sources) : null,
+				JSON.stringify(generation.result),
 			);
 	}
 

package/src/cli.ts

@@ -12,6 +12,7 @@ import { serveMain } from "./daemon.ts";
 import type { JobSnapshot } from "./domain/bounded-job-executor.ts";
 import type { ContentHash } from "./domain/content-hash.ts";
 import { DEFAULT_PACKAGE_SOURCE_BOUNDS, type PackageSourceOperationResult } from "./domain/package-source.ts";
+import type { PopulateSymbolGraphResult } from "./domain/populate-symbol-graph.ts";
 import type { SymbolSearchResult } from "./domain/workspace-symbol.ts";
 import type { WorkspacePort } from "./ports/workspace-port.ts";
 import type { WorkspaceId } from "./service.ts";
@@ -361,11 +362,19 @@ function requiredIntFlag(flags: string[], flag: string): number {
 	return parsed;
 }
 
-function formatJobSnapshot(job: JobSnapshot<{ filesProcessed: number; symbolsProcessed: number; nodesAdded: number; edgesAdded: number }>): string {
+function formatPopulationResult(result: PopulateSymbolGraphResult): string {
+	const counts = `${result.filesProcessed}/${result.filesAttempted} files, ${result.symbolsProcessed} symbols, ${result.nodesAdded} nodes, ${result.edgesAdded} edges`;
+	if (result.completeness === "complete") return counts;
+	const first = result.failures[0];
+	const failure = first ? `; first failure: ${first.path} [${first.code} via ${first.provenance.backend}] ${first.message}` : "";
+	return `partial -- ${counts}, ${result.filesFailed} failed files (${result.failureCount} failed operations)${failure}`;
+}
+
+function formatJobSnapshot(job: JobSnapshot<PopulateSymbolGraphResult>): string {
 	if (job.status === "queued") return `${job.id}: queued (${job.operation}); poll with: lector job status ${job.id}`;
 	if (job.status === "running") return `${job.id}: still running (${job.operation}); poll with: lector job status ${job.id}`;
 	if (job.status === "failed") return `${job.id}: failed [${job.error.code}] -- ${job.error.message}`;
-	return `${job.id}: succeeded -- ${job.result.filesProcessed} files, ${job.result.symbolsProcessed} symbols, ${job.result.nodesAdded} nodes, ${job.result.edgesAdded} edges`;
+	return `${job.id}: succeeded -- ${formatPopulationResult(job.result)}`;
 }
 
 async function runWorkspacePopulateSymbolGraph(workspaceId: string | undefined, flags: string[]): Promise<void> {
@@ -385,11 +394,7 @@ async function runWorkspacePopulateSymbolGraph(workspaceId: string | undefined,
 		return;
 	}
 	const result = await client.call("workspace.populateSymbolGraph", { workspaceId, maxFiles, maxSymbolsPerFile });
-	console.log(
-		hasFlag(flags, "--json")
-			? JSON.stringify(result)
-			: `${result.filesProcessed} files, ${result.symbolsProcessed} symbols, ${result.nodesAdded} nodes, ${result.edgesAdded} edges`,
-	);
+	console.log(hasFlag(flags, "--json") ? JSON.stringify(result) : formatPopulationResult(result));
 }
 
 async function runJobStatus(jobId: string | undefined, flags: string[]): Promise<void> {
@@ -411,6 +416,7 @@ async function runWorkspaceCacheStatus(workspaceId: string | undefined, flags: s
 	}
 	if (status.status === "not-cached") console.log(`not cached -- ${status.reason}`);
 	else if (status.status === "caching") console.log(`caching -- job ${status.jobId}`);
+	else if (status.status === "partial") console.log(`partially cached -- ${formatPopulationResult(status.generation.result)}`);
 	else console.log(`cached -- completed ${new Date(status.generation.completedAt).toISOString()}`);
 }
 

package/src/domain/populate-symbol-graph.ts

@@ -1,17 +1,60 @@
 import type { CodeIntelligencePort } from "../ports/code-intelligence-port.ts";
 import type { SymbolGraphPort, SymbolNode } from "../ports/symbol-graph-port.ts";
+import type { OutgoingCall } from "./call-hierarchy.ts";
 import type { DocumentSymbolEntry } from "./document-symbol.ts";
+import type { IntelligenceProvenance } from "./intelligence-provenance.ts";
 import { deriveSymbolNodeId } from "./symbol-node-id.ts";
 import type { WorkspaceLocation } from "./workspace-symbol.ts";
 
 const CALLABLE_KINDS = new Set(["function", "method", "constructor"]);
+const MAX_RECORDED_FAILURES = 100;
+const MAX_FAILURE_MESSAGE_LENGTH = 500;
+
+export interface SymbolGraphPopulationFailure {
+	readonly path: string;
+	readonly operation: "document-symbols" | "outgoing-calls";
+	readonly code: string;
+	readonly message: string;
+	readonly provenance: IntelligenceProvenance;
+}
 
 export interface PopulateSymbolGraphResult {
+	readonly completeness: "complete" | "partial";
+	readonly filesAttempted: number;
 	readonly filesProcessed: number;
+	readonly filesFailed: number;
 	readonly symbolsProcessed: number;
 	/** addNode calls made, not necessarily new nodes -- a symbol reached from multiple edges is upserted once per encounter within a run, deduped in-memory. */
 	readonly nodesAdded: number;
 	readonly edgesAdded: number;
+	readonly failureCount: number;
+	readonly failures: readonly SymbolGraphPopulationFailure[];
+	readonly failuresTruncated: boolean;
+}
+
+const UNKNOWN_PROVENANCE: IntelligenceProvenance = {
+	fidelity: "semantic",
+	backend: "unavailable",
+	languageId: "unknown",
+	authority: "language-server",
+	freshness: "live-process",
+	limitations: ["source provenance was unavailable"],
+};
+
+function boundedFailure(
+	index: CodeIntelligencePort,
+	path: string,
+	operation: SymbolGraphPopulationFailure["operation"],
+	error: unknown,
+): SymbolGraphPopulationFailure {
+	const errorName = error instanceof Error ? error.name : undefined;
+	return {
+		path,
+		operation,
+		code: errorName && errorName !== "Error" ? errorName : "CodeIntelligenceFileError",
+		message: (error instanceof Error ? error.message : String(error)).slice(0, MAX_FAILURE_MESSAGE_LENGTH),
+		provenance: index.provenanceForPath?.(path) ?? index.provenance ?? UNKNOWN_PROVENANCE,
+	};
 }
 
 function toLocation(entry: DocumentSymbolEntry): WorkspaceLocation {
@@ -50,7 +93,16 @@ export async function populateSymbolGraph(
 	let symbolsProcessed = 0;
 	let nodesAdded = 0;
 	let edgesAdded = 0;
+	let failureCount = 0;
 	const addedNodeIds = new Set<string>();
+	const failedFiles = new Set<string>();
+	const failures: SymbolGraphPopulationFailure[] = [];
+
+	function recordFailure(file: string, operation: SymbolGraphPopulationFailure["operation"], error: unknown): void {
+		failureCount++;
+		failedFiles.add(file);
+		if (failures.length < MAX_RECORDED_FAILURES) failures.push(boundedFailure(index, file, operation, error));
+	}
 
 	async function ensureNode(node: SymbolNode): Promise<void> {
 		if (addedNodeIds.has(node.id)) return;
@@ -60,7 +112,13 @@ export async function populateSymbolGraph(
 	}
 
 	for (const file of files) {
-		const topLevel = await index.documentSymbols(file);
+		let topLevel: DocumentSymbolEntry[];
+		try {
+			topLevel = await index.documentSymbols(file);
+		} catch (error) {
+			recordFailure(file, "document-symbols", error);
+			continue;
+		}
 		const flattened = flattenDocumentSymbols(topLevel).slice(0, maxSymbolsPerFile);
 		filesProcessed++;
 
@@ -76,7 +134,13 @@ export async function populateSymbolGraph(
 			}
 
 			if (CALLABLE_KINDS.has(entry.kind)) {
-				const callees = await index.outgoingCalls(location);
+				let callees: OutgoingCall[];
+				try {
+					callees = await index.outgoingCalls(location);
+				} catch (error) {
+					recordFailure(file, "outgoing-calls", error);
+					continue;
+				}
 				for (const call of callees) {
 					const calleeNode: SymbolNode = { id: deriveSymbolNodeId(call.to.location), name: call.to.name, kind: call.to.kind, location: call.to.location };
 					await ensureNode(calleeNode);
@@ -87,5 +151,16 @@ export async function populateSymbolGraph(
 		}
 	}
 
-	return { filesProcessed, symbolsProcessed, nodesAdded, edgesAdded };
+	return {
+		completeness: failureCount === 0 ? "complete" : "partial",
+		filesAttempted: files.length,
+		filesProcessed,
+		filesFailed: failedFiles.size,
+		symbolsProcessed,
+		nodesAdded,
+		edgesAdded,
+		failureCount,
+		failures,
+		failuresTruncated: failureCount > failures.length,
+	};
 }

package/src/domain/symbol-graph-generation.ts

@@ -16,4 +16,5 @@ export interface SymbolGraphGeneration {
 export type WorkspaceCacheStatus =
 	| { readonly status: "not-cached"; readonly reason: "no-completed-generation" | "bounds-changed" | "source-changed" }
 	| { readonly status: "caching"; readonly jobId: string }
+	| { readonly status: "partial"; readonly generation: SymbolGraphGeneration }
 	| { readonly status: "cached"; readonly generation: SymbolGraphGeneration };

package/src/index.ts

@@ -130,7 +130,7 @@ export type {
 	VerifiedPackageSource,
 } from "./domain/package-source.ts";
 export { DEFAULT_PACKAGE_SOURCE_BOUNDS } from "./domain/package-source.ts";
-export { type PopulateSymbolGraphResult, populateSymbolGraph } from "./domain/populate-symbol-graph.ts";
+export { type PopulateSymbolGraphResult, populateSymbolGraph, type SymbolGraphPopulationFailure } from "./domain/populate-symbol-graph.ts";
 export { prepareCallHierarchy } from "./domain/prepare-call-hierarchy.ts";
 export { raceWorkspaceQuery } from "./domain/race-workspace-query.ts";
 export { type RawRead, rawRead, WorkspaceEntryNotFound } from "./domain/raw-read.ts";

package/src/ports/code-intelligence-port.ts

@@ -2,6 +2,7 @@ import type { CallHierarchyEntry, IncomingCall, OutgoingCall } from "../domain/c
 import type { Diagnostic } from "../domain/diagnostic.ts";
 import type { DocumentSymbolEntry } from "../domain/document-symbol.ts";
 import type { Hover } from "../domain/hover.ts";
+import type { IntelligenceProvenance } from "../domain/intelligence-provenance.ts";
 import type { WorkspaceLocation } from "../domain/workspace-symbol.ts";
 
 /**
@@ -17,6 +18,8 @@ import type { WorkspaceLocation } from "../domain/workspace-symbol.ts";
  * process findSymbols already keeps alive per workspace -- not a second one.
  */
 export interface CodeIntelligencePort {
+	readonly provenance?: IntelligenceProvenance;
+	provenanceForPath?(path: string): IntelligenceProvenance | undefined;
 	/** Where the symbol at `at` is actually declared -- may cross files, may return more than one candidate. */
 	goToDefinition(at: WorkspaceLocation): Promise<WorkspaceLocation[]>;
 	/** Every concrete implementation of the interface/abstract member at `at` -- unlike goToDefinition, crosses a port/interface boundary into its real adapters. */

package/src/service.ts

@@ -313,7 +313,7 @@ export interface OperationOutputs {
 	"workspace.prepareCallHierarchy": Provenanced<{ items: readonly CallHierarchyEntry[] }>;
 	"workspace.incomingCalls": Provenanced<{ calls: readonly IncomingCall[] }>;
 	"workspace.outgoingCalls": Provenanced<{ calls: readonly OutgoingCall[] }>;
-	"workspace.populateSymbolGraph": { filesProcessed: number; symbolsProcessed: number; nodesAdded: number; edgesAdded: number };
+	"workspace.populateSymbolGraph": PopulateSymbolGraphResult;
 	"workspace.reachableFrom": { symbols: readonly SymbolNode[] };
 	"workspace.symbolEdgesFrom": { symbols: readonly SymbolNode[] };
 	"workspace.symbolEdgesTo": { symbols: readonly SymbolNode[] };
@@ -911,7 +911,7 @@ export function createLectorService(workspaces: ReadonlyMap<WorkspaceId, Workspa
 			return { status: "not-cached", reason: "source-changed" };
 		}
 		if (currentFingerprint !== generation.sourceFingerprint) return { status: "not-cached", reason: "source-changed" };
-		return { status: "cached", generation };
+		return generation.result.completeness === "partial" ? { status: "partial", generation } : { status: "cached", generation };
 	}
 
 	function isRecord(value: unknown): value is Record<string, unknown> {