@gmickel/gno 1.19.0 → 1.21.0

package/src/ingestion/sync.ts

@@ -40,6 +40,13 @@ import {
   getDefaultPipeline,
 } from "../converters/pipeline";
 import { DEFAULT_LIMITS } from "../converters/types";
+import {
+  diffDocumentStructure,
+  extractDocumentStructure,
+  isRelationMap,
+  normalizeRelationEdgeType,
+  normalizeRelationTarget,
+} from "../core/change-diff";
 import {
   normalizeMarkdownPath,
   normalizeWikiName,
@@ -83,35 +90,6 @@ const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([
   "UNSUPPORTED",
 ]);
 
-type RelationMap = Record<string, string[]>;
-
-function isRelationMap(value: unknown): value is RelationMap {
-  if (!value || typeof value !== "object" || Array.isArray(value)) {
-    return false;
-  }
-
-  return Object.values(value).every(
-    (targets) =>
-      Array.isArray(targets) &&
-      targets.every((target) => typeof target === "string")
-  );
-}
-
-function normalizeRelationTarget(raw: string): string {
-  const trimmed = raw.trim();
-  if (trimmed.startsWith("[[") && trimmed.endsWith("]]")) {
-    return trimmed.slice(2, -2).split("|")[0]?.trim() ?? "";
-  }
-  return trimmed;
-}
-
-function normalizeRelationEdgeType(raw: string): string {
-  return raw
-    .trim()
-    .toLowerCase()
-    .replace(/[-\s]+/g, "_");
-}
-
 function findDocByWikiRef(
   docs: DocumentRow[],
   targetRef: string,
@@ -747,7 +725,24 @@ export class SyncService {
           // Log but continue - error recording is best-effort
         }
 
-        // Upsert document with error info, explicitly clear mirrorHash
+        const hadRetrievableEvidence = Boolean(existing?.mirrorHash);
+        const structureDelta = hadRetrievableEvidence
+          ? diffDocumentStructure(
+              await this.readPreviousStructure(store, existing),
+              {
+                headings: [],
+                links: [],
+                typedEdges: [],
+                dates: {},
+              }
+            ).delta
+          : undefined;
+
+        // Upsert document with error info, explicitly clear mirrorHash. This
+        // transition is journaled only when retrievable evidence previously
+        // existed; initial/repeated error placeholders are not evidence
+        // creates. Real evidence disappearance stays in the same transaction
+        // so saved Capsule freshness checks cannot miss it.
         const upsertResult = await store.upsertDocument({
           collection: collection.name,
           relPath: entry.relPath,
@@ -761,6 +756,7 @@ export class SyncService {
           lastErrorMessage: convertResult.error.message,
           ingestVersion: INGEST_VERSION,
           contentTypeRulesFingerprint,
+          changeJournal: structureDelta ? { structureDelta } : false,
           // mirrorHash intentionally omitted (will be null)
         });
 
@@ -788,155 +784,186 @@ export class SyncService {
         mime.ext,
         contentTypeRules
       );
-
-      // 7. Upsert document - EXPLICITLY clear error fields on success
-      const docidResult = await store.upsertDocument({
-        collection: collection.name,
-        relPath: entry.relPath,
-        sourceHash,
-        sourceMime: mime.mime,
-        sourceExt: mime.ext,
-        sourceSize,
-        sourceMtime,
-        sourceCtime,
-        title: artifact.title,
-        mirrorHash: artifact.mirrorHash,
-        converterId: artifact.meta.converterId,
-        converterVersion: artifact.meta.converterVersion,
-        languageHint: artifact.languageHint ?? collection.languageHint,
-        contentType: extractedMetadata.contentType,
-        contentTypeSource: extractedMetadata.contentTypeSource,
-        categories: extractedMetadata.categories,
-        author: extractedMetadata.author,
-        frontmatterDate: extractedMetadata.frontmatterDate,
-        dateFields: extractedMetadata.dateFields,
-        contentTypeRulesFingerprint,
-        // Clear error fields on success (requires store to handle undefined → null)
-        lastErrorCode: undefined,
-        lastErrorMessage: undefined,
-        ingestVersion: INGEST_VERSION,
-      });
-
-      const { id: docId, docid } = mustOk(docidResult, "upsertDocument", {
-        collection: collection.name,
-        relPath: entry.relPath,
-      });
-
-      // 8. Upsert content (content-addressed dedupe) - CHECKED
-      const contentResult = await store.upsertContent(
-        artifact.mirrorHash,
-        artifact.markdown
+      const previousStructure = await this.readPreviousStructure(
+        store,
+        existing
       );
-      mustOk(contentResult, "upsertContent", {
-        mirrorHash: artifact.mirrorHash,
-      });
-
-      // 9. Chunk content
-      const chunks = this.chunker.chunk(
+      const nextStructure = extractDocumentStructure(
         artifact.markdown,
-        DEFAULT_CHUNK_PARAMS,
-        artifact.languageHint ?? collection.languageHint,
-        entry.relPath
+        entry.relPath,
+        extractedMetadata.dateFields
       );
+      const structureDelta = diffDocumentStructure(
+        previousStructure,
+        nextStructure
+      ).delta;
+
+      const persistSuccessfulFile = async (): Promise<FileSyncResult> => {
+        // 7. Upsert document - EXPLICITLY clear error fields on success
+        const docidResult = await store.upsertDocument({
+          collection: collection.name,
+          relPath: entry.relPath,
+          sourceHash,
+          sourceMime: mime.mime,
+          sourceExt: mime.ext,
+          sourceSize,
+          sourceMtime,
+          sourceCtime,
+          title: artifact.title,
+          mirrorHash: artifact.mirrorHash,
+          converterId: artifact.meta.converterId,
+          converterVersion: artifact.meta.converterVersion,
+          languageHint: artifact.languageHint ?? collection.languageHint,
+          contentType: extractedMetadata.contentType,
+          contentTypeSource: extractedMetadata.contentTypeSource,
+          categories: extractedMetadata.categories,
+          author: extractedMetadata.author,
+          frontmatterDate: extractedMetadata.frontmatterDate,
+          dateFields: extractedMetadata.dateFields,
+          contentTypeRulesFingerprint,
+          // Clear error fields on success (requires store to handle undefined → null)
+          lastErrorCode: undefined,
+          lastErrorMessage: undefined,
+          ingestVersion: INGEST_VERSION,
+          changeJournal: { structureDelta },
+        });
 
-      // 10. Convert to ChunkInput for store
-      const chunkInputs: ChunkInput[] = chunks.map((c) => ({
-        seq: c.seq,
-        pos: c.pos,
-        text: c.text,
-        startLine: c.startLine,
-        endLine: c.endLine,
-        language: c.language ?? undefined,
-        tokenCount: c.tokenCount ?? undefined,
-      }));
-
-      // 11. Upsert chunks - CHECKED
-      const chunksResult = await store.upsertChunks(
-        artifact.mirrorHash,
-        chunkInputs
-      );
-      mustOk(chunksResult, "upsertChunks", {
-        mirrorHash: artifact.mirrorHash,
-        chunkCount: chunkInputs.length,
-      });
+        const { id: docId, docid } = mustOk(docidResult, "upsertDocument", {
+          collection: collection.name,
+          relPath: entry.relPath,
+        });
 
-      // 12. Rebuild FTS for this hash - CHECKED
-      const ftsResult = await store.rebuildFtsForHash(artifact.mirrorHash);
-      mustOk(ftsResult, "rebuildFtsForHash", {
-        mirrorHash: artifact.mirrorHash,
-      });
+        // 8. Upsert content (content-addressed dedupe) - CHECKED
+        const contentResult = await store.upsertContent(
+          artifact.mirrorHash,
+          artifact.markdown
+        );
+        mustOk(contentResult, "upsertContent", {
+          mirrorHash: artifact.mirrorHash,
+        });
 
-      // 13. Extract and store tags from frontmatter and body hashtags
-      // Always call setDocTags to clear removed tags on re-sync
-      const extractedTags = extractTags(artifact.markdown);
-      const tagsResult = await store.setDocTags(
-        docId,
-        extractedTags,
-        "frontmatter"
-      );
-      mustOk(tagsResult, "setDocTags", {
-        docId,
-        tagCount: extractedTags.length,
-      });
+        // 9. Chunk content
+        const chunks = this.chunker.chunk(
+          artifact.markdown,
+          DEFAULT_CHUNK_PARAMS,
+          artifact.languageHint ?? collection.languageHint,
+          entry.relPath
+        );
 
-      // 14. Extract and store links (wiki and markdown links)
-      const excludedRanges = getExcludedRanges(artifact.markdown);
-      const lineOffsets = buildLineOffsets(artifact.markdown);
-      const parsedLinks = parseLinks(
-        artifact.markdown,
-        lineOffsets,
-        excludedRanges
-      );
+        // 10. Convert to ChunkInput for store
+        const chunkInputs: ChunkInput[] = chunks.map((c) => ({
+          seq: c.seq,
+          pos: c.pos,
+          text: c.text,
+          startLine: c.startLine,
+          endLine: c.endLine,
+          language: c.language ?? undefined,
+          tokenCount: c.tokenCount ?? undefined,
+        }));
+
+        // 11. Upsert chunks - CHECKED
+        const chunksResult = await store.upsertChunks(
+          artifact.mirrorHash,
+          chunkInputs
+        );
+        mustOk(chunksResult, "upsertChunks", {
+          mirrorHash: artifact.mirrorHash,
+          chunkCount: chunkInputs.length,
+        });
 
-      const linkInputs: DocLinkInput[] = [];
-      for (const link of parsedLinks) {
-        // Compute target_ref_norm based on link type
-        let targetRefNorm: string;
-        if (link.kind === "wiki") {
-          targetRefNorm = normalizeWikiName(link.targetRef);
-        } else {
-          // Markdown links with collection prefix are not supported
-          // (use wiki links for cross-collection references)
-          if (link.targetCollection) {
-            continue;
-          }
-          const resolved = normalizeMarkdownPath(link.targetRef, entry.relPath);
-          if (!resolved) {
-            // Link escapes collection root - skip silently
-            continue;
+        // 12. Rebuild FTS for this hash - CHECKED
+        const ftsResult = await store.rebuildFtsForHash(artifact.mirrorHash);
+        mustOk(ftsResult, "rebuildFtsForHash", {
+          mirrorHash: artifact.mirrorHash,
+        });
+
+        // 13. Extract and store tags from frontmatter and body hashtags
+        // Always call setDocTags to clear removed tags on re-sync
+        const extractedTags = extractTags(artifact.markdown);
+        const tagsResult = await store.setDocTags(
+          docId,
+          extractedTags,
+          "frontmatter"
+        );
+        mustOk(tagsResult, "setDocTags", {
+          docId,
+          tagCount: extractedTags.length,
+        });
+
+        // 14. Extract and store links (wiki and markdown links)
+        const excludedRanges = getExcludedRanges(artifact.markdown);
+        const lineOffsets = buildLineOffsets(artifact.markdown);
+        const parsedLinks = parseLinks(
+          artifact.markdown,
+          lineOffsets,
+          excludedRanges
+        );
+
+        const linkInputs: DocLinkInput[] = [];
+        for (const link of parsedLinks) {
+          // Compute target_ref_norm based on link type
+          let targetRefNorm: string;
+          if (link.kind === "wiki") {
+            targetRefNorm = normalizeWikiName(link.targetRef);
+          } else {
+            // Markdown links with collection prefix are not supported
+            // (use wiki links for cross-collection references)
+            if (link.targetCollection) {
+              continue;
+            }
+            const resolved = normalizeMarkdownPath(
+              link.targetRef,
+              entry.relPath
+            );
+            if (!resolved) {
+              // Link escapes collection root - skip silently
+              continue;
+            }
+            targetRefNorm = resolved;
           }
-          targetRefNorm = resolved;
+
+          linkInputs.push({
+            targetRef: link.targetRef,
+            targetRefNorm,
+            targetAnchor: link.targetAnchor,
+            targetCollection: link.targetCollection,
+            linkType: link.kind,
+            linkText: link.displayText,
+            startLine: link.startLine,
+            startCol: link.startCol,
+            endLine: link.endLine,
+            endCol: link.endCol,
+          });
         }
 
-        linkInputs.push({
-          targetRef: link.targetRef,
-          targetRefNorm,
-          targetAnchor: link.targetAnchor,
-          targetCollection: link.targetCollection,
-          linkType: link.kind,
-          linkText: link.displayText,
-          startLine: link.startLine,
-          startCol: link.startCol,
-          endLine: link.endLine,
-          endCol: link.endCol,
+        const linksResult = await store.setDocLinks(
+          docId,
+          linkInputs,
+          "parsed"
+        );
+        mustOk(linksResult, "setDocLinks", {
+          docId,
+          linkCount: linkInputs.length,
         });
-      }
 
-      const linksResult = await store.setDocLinks(docId, linkInputs, "parsed");
-      mustOk(linksResult, "setDocLinks", {
-        docId,
-        linkCount: linkInputs.length,
-      });
-
-      const status = existing ? "updated" : "added";
-      return {
-        relPath: entry.relPath,
-        status,
-        docid,
-        mirrorHash: artifact.mirrorHash,
-        contentType: extractedMetadata.contentType,
-        contentTypeSource: extractedMetadata.contentTypeSource,
+        const status = existing ? "updated" : "added";
+        return {
+          relPath: entry.relPath,
+          status,
+          docid,
+          mirrorHash: artifact.mirrorHash,
+          contentType: extractedMetadata.contentType,
+          contentTypeSource: extractedMetadata.contentTypeSource,
+        };
       };
+      if (!store.withTransaction) {
+        return await persistSuccessfulFile();
+      }
+      const persisted = await store.withTransaction(persistSuccessfulFile);
+      return mustOk(persisted, "persistSuccessfulFile", {
+        collection: collection.name,
+        relPath: entry.relPath,
+      });
     } catch (error) {
       const message = error instanceof Error ? error.message : "Unknown error";
       // Distinguish store errors from other internal errors
@@ -977,6 +1004,7 @@ export class SyncService {
               existingResult.value.sourceMtime,
             lastErrorCode: code,
             lastErrorMessage: message,
+            changeJournal: false,
           });
         }
       } catch {
@@ -992,6 +1020,27 @@ export class SyncService {
     }
   }
 
+  private async readPreviousStructure(
+    store: StorePort,
+    existing: DocumentRow | null
+  ): Promise<ReturnType<typeof extractDocumentStructure> | null | undefined> {
+    if (!existing) return null;
+    if (!existing.mirrorHash) return undefined;
+
+    const content = await store.getContent(existing.mirrorHash);
+    if (!content.ok) {
+      throw new Error(`Store operation failed: ${content.error.message}`, {
+        cause: content.error,
+      });
+    }
+    if (content.value === null) return undefined;
+    return extractDocumentStructure(
+      content.value,
+      existing.relPath,
+      existing.dateFields
+    );
+  }
+
   /**
    * Sync a specific set of files within a collection.
    */

package/src/llm/errors.ts

@@ -22,7 +22,8 @@ export type LlmErrorCode =
   | "OUT_OF_MEMORY"
   | "INVALID_URI"
   | "LOCK_FAILED"
-  | "AUTO_DOWNLOAD_DISABLED";
+  | "AUTO_DOWNLOAD_DISABLED"
+  | "STRUCTURED_OUTPUT_UNAVAILABLE";
 
 export interface LlmError {
   code: LlmErrorCode;
@@ -248,3 +249,11 @@ export function autoDownloadDisabledError(uri: string): LlmError {
     suggestion: "Run 'gno models pull' to download models manually.",
   });
 }
+
+export function structuredOutputUnavailableError(uri: string): LlmError {
+  return llmError("STRUCTURED_OUTPUT_UNAVAILABLE", {
+    message: `JSON Schema constrained generation is unavailable for model: ${uri}`,
+    modelUri: uri,
+    retryable: false,
+  });
+}

package/src/llm/httpGeneration.ts

@@ -7,7 +7,10 @@
 
 import type { GenerationPort, GenParams, LlmResult } from "./types";
 
-import { inferenceFailedError } from "./errors";
+import {
+  inferenceFailedError,
+  structuredOutputUnavailableError,
+} from "./errors";
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Types
@@ -42,6 +45,7 @@ export class HttpGeneration implements GenerationPort {
   private readonly apiUrl: string;
   private readonly modelName: string;
   readonly modelUri: string;
+  readonly structuredOutput = "none" as const;
 
   constructor(modelUri: string) {
     this.modelUri = modelUri;
@@ -63,6 +67,12 @@ export class HttpGeneration implements GenerationPort {
     prompt: string,
     params?: GenParams
   ): Promise<LlmResult<string>> {
+    if (params?.jsonSchema) {
+      return {
+        ok: false,
+        error: structuredOutputUnavailableError(this.modelUri),
+      };
+    }
     try {
       const response = await fetch(this.apiUrl, {
         method: "POST",

package/src/llm/nodeLlamaCpp/generation.ts

@@ -18,6 +18,39 @@ type LlamaModel = Awaited<
     Awaited<ReturnType<typeof import("node-llama-cpp").getLlama>>["loadModel"]
   >
 >;
+type Llama = Awaited<ReturnType<typeof import("node-llama-cpp").getLlama>>;
+type JsonGrammarSchema = Parameters<Llama["createGrammarForJsonSchema"]>[0];
+
+export interface JsonSchemaGrammarLike {
+  parse(response: string): unknown;
+}
+
+export interface StructuredPromptSession {
+  prompt(
+    prompt: string,
+    options: {
+      temperature: number;
+      seed: number;
+      maxTokens: number;
+      grammar?: JsonSchemaGrammarLike;
+    }
+  ): Promise<string>;
+}
+
+export const promptWithJsonSchemaGrammar = async (
+  session: StructuredPromptSession,
+  prompt: string,
+  options: {
+    temperature: number;
+    seed: number;
+    maxTokens: number;
+  },
+  grammar?: JsonSchemaGrammarLike
+): Promise<string> => {
+  const response = await session.prompt(prompt, { ...options, grammar });
+  grammar?.parse(response);
+  return response;
+};
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Default Parameters (for determinism)
@@ -34,6 +67,7 @@ const DEFAULT_MAX_TOKENS = 256;
 export class NodeLlamaCppGeneration implements GenerationPort {
   private readonly manager: ModelManager;
   readonly modelUri: string;
+  readonly structuredOutput = "json_schema" as const;
   private readonly modelPath: string;
 
   constructor(manager: ModelManager, modelUri: string, modelPath: string) {
@@ -56,11 +90,16 @@ export class NodeLlamaCppGeneration implements GenerationPort {
     }
 
     const llamaModel = model.value.model as LlamaModel;
-    const context = await llamaModel.createContext(
-      params?.contextSize ? { contextSize: params.contextSize } : undefined
-    );
-
+    let context: Awaited<ReturnType<LlamaModel["createContext"]>> | null = null;
     try {
+      const grammar = params?.jsonSchema
+        ? await (
+            await this.manager.getLlama()
+          ).createGrammarForJsonSchema(params.jsonSchema as JsonGrammarSchema)
+        : undefined;
+      context = await llamaModel.createContext(
+        params?.contextSize ? { contextSize: params.contextSize } : undefined
+      );
       // Import LlamaChatSession dynamically
       const { LlamaChatSession } = await import("node-llama-cpp");
       const session = new LlamaChatSession({
@@ -68,17 +107,22 @@ export class NodeLlamaCppGeneration implements GenerationPort {
       });
 
       // Note: stop sequences not yet supported - requires stopOnTrigger API
-      const response = await session.prompt(prompt, {
-        temperature: params?.temperature ?? DEFAULT_TEMPERATURE,
-        seed: params?.seed ?? DEFAULT_SEED,
-        maxTokens: params?.maxTokens ?? DEFAULT_MAX_TOKENS,
-      });
+      const response = await promptWithJsonSchemaGrammar(
+        session as StructuredPromptSession,
+        prompt,
+        {
+          temperature: params?.temperature ?? DEFAULT_TEMPERATURE,
+          seed: params?.seed ?? DEFAULT_SEED,
+          maxTokens: params?.maxTokens ?? DEFAULT_MAX_TOKENS,
+        },
+        grammar
+      );
 
       return { ok: true, value: response };
     } catch (e) {
       return { ok: false, error: inferenceFailedError(this.modelUri, e) };
     } finally {
-      await context.dispose().catch(() => {
+      await context?.dispose().catch(() => {
         // Ignore disposal errors
       });
     }

package/src/llm/types.ts

@@ -58,8 +58,12 @@ export interface GenParams {
   contextSize?: number;
   /** Stop sequences */
   stop?: string[];
+  /** Closed JSON Schema enforced by a capable generation backend. */
+  jsonSchema?: Readonly<Record<string, unknown>>;
 }
 
+export type StructuredOutputCapability = "json_schema" | "none";
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Rerank Types
 // ─────────────────────────────────────────────────────────────────────────────
@@ -90,6 +94,8 @@ export interface EmbeddingPort {
 
 export interface GenerationPort {
   readonly modelUri: string;
+  /** Undefined is treated as unsupported for backwards-compatible ports. */
+  readonly structuredOutput?: StructuredOutputCapability;
   generate(prompt: string, params?: GenParams): Promise<LlmResult<string>>;
   dispose(): Promise<void>;
 }