@gmickel/gno 1.7.1 → 1.9.0

package/src/serve/routes/api.ts

@@ -5,10 +5,10 @@
  * @module src/serve/routes/api
  */
 
-// node:fs/promises structure walk has no Bun equivalent
-import { readdir } from "node:fs/promises";
+// node:fs/promises structure ops have no Bun equivalent
+import { mkdir, readdir } from "node:fs/promises";
 // node:path has no Bun equivalent
-import { posix as pathPosix } from "node:path";
+import { dirname, join as pathJoin, posix as pathPosix } from "node:path";
 
 import type {
   Collection,
@@ -35,6 +35,14 @@ import {
   removeCollection,
   updateCollection,
 } from "../../collection";
+import {
+  buildCaptureReceipt,
+  listCaptureDiskRelPaths,
+  planCapture,
+  type CapturePlan,
+  type PublicCaptureInput,
+} from "../../core/capture";
+import { writeCapturePlanFile } from "../../core/capture-write";
 import {
   buildEditableCopyContent,
   deriveEditableCopyRelPath,
@@ -74,7 +82,11 @@ import {
   validateTag,
 } from "../../core/tags";
 import { validateRelPath } from "../../core/validation";
-import { defaultSyncService, type SyncResult } from "../../ingestion";
+import {
+  defaultSyncService,
+  type SyncResult,
+  withContentTypeRules,
+} from "../../ingestion";
 import { updateFrontmatterTags } from "../../ingestion/frontmatter";
 import {
   getCollectionEffectiveModels,
@@ -292,6 +304,8 @@ export interface CreateDocRequestBody {
   tags?: string[];
 }
 
+export interface CreateCaptureRequestBody extends PublicCaptureInput {}
+
 export interface RenameDocRequestBody {
   name: string;
   uri?: string;
@@ -909,10 +923,17 @@ export async function handleCreateCollection(
     return errorResponse("RUNTIME", "Collection not found after add", 500);
   }
   const jobResult = startJob("add", async (): Promise<SyncResult> => {
-    const result = await defaultSyncService.syncCollection(collection, store, {
-      gitPull: body.gitPull,
-      runUpdateCmd: true,
-    });
+    const result = await defaultSyncService.syncCollection(
+      collection,
+      store,
+      withContentTypeRules(
+        {
+          gitPull: body.gitPull,
+          runUpdateCmd: true,
+        },
+        syncResult.config
+      )
+    );
     if (result.filesAdded > 0 || result.filesUpdated > 0) {
       if (ctxHolder.scheduler) {
         await ctxHolder.scheduler.triggerNow();
@@ -1204,10 +1225,17 @@ export async function handleSync(
 
   // Start background sync job
   const jobResult = startJob("sync", async (): Promise<SyncResult> => {
-    const result = await defaultSyncService.syncAll(collections, store, {
-      gitPull: body.gitPull,
-      runUpdateCmd: true,
-    });
+    const result = await defaultSyncService.syncAll(
+      collections,
+      store,
+      withContentTypeRules(
+        {
+          gitPull: body.gitPull,
+          runUpdateCmd: true,
+        },
+        ctxHolder.config
+      )
+    );
     if (result.totalFilesAdded > 0 || result.totalFilesUpdated > 0) {
       if (ctxHolder.scheduler) {
         await ctxHolder.scheduler.triggerNow();
@@ -1833,9 +1861,11 @@ export async function handleRenameDoc(
     const nextUri = `gno://${collection.name}/${nextRelPath}`;
     let warning: string | undefined;
     try {
-      await syncCollection(collection, store, {
-        runUpdateCmd: false,
-      });
+      await syncCollection(
+        collection,
+        store,
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
+      );
     } catch {
       warning =
         "File renamed on disk, but index refresh failed. Run Update All to reconcile the workspace.";
@@ -2053,9 +2083,11 @@ export async function handleTrashDoc(
     }
     let warning: string | undefined;
     try {
-      await syncCollection(collection, store, {
-        runUpdateCmd: false,
-      });
+      await syncCollection(
+        collection,
+        store,
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
+      );
     } catch {
       warning =
         "File moved to Trash, but index refresh failed. Run Update All to reconcile the workspace.";
@@ -2174,9 +2206,11 @@ export async function handleMoveDoc(
     await renameFilePath(fullPath, nextFullPath);
     let warning: string | undefined;
     try {
-      await defaultSyncService.syncCollection(collection, store, {
-        runUpdateCmd: false,
-      });
+      await defaultSyncService.syncCollection(
+        collection,
+        store,
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
+      );
     } catch {
       warning =
         "File moved on disk, but index refresh failed. Run Update All to reconcile the workspace.";
@@ -2295,9 +2329,11 @@ export async function handleDuplicateDoc(
     await copyFilePath(fullPath, nextFullPath);
     let warning: string | undefined;
     try {
-      await defaultSyncService.syncCollection(collection, store, {
-        runUpdateCmd: false,
-      });
+      await defaultSyncService.syncCollection(
+        collection,
+        store,
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
+      );
     } catch {
       warning =
         "File duplicated on disk, but index refresh failed. Run Update All to reconcile the workspace.";
@@ -2626,7 +2662,7 @@ export async function handleUpdateDoc(
         const result = await defaultSyncService.syncCollection(
           collection,
           store,
-          { runUpdateCmd: false }
+          withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
         );
         // Notify scheduler after sync completes
         ctxHolder.scheduler?.notifySyncComplete([doc.docid]);
@@ -2795,6 +2831,157 @@ export async function handleCreateEditableCopy(
   return handleCreateDoc(ctxHolder, store, createReq);
 }
 
+/**
+ * POST /api/capture
+ * Capture a note with structured provenance.
+ */
+export async function handleCreateCapture(
+  ctxHolder: ContextHolder,
+  store: SqliteAdapter,
+  req: Request
+): Promise<Response> {
+  let body: CreateCaptureRequestBody;
+  try {
+    body = (await req.json()) as CreateCaptureRequestBody;
+  } catch {
+    return errorResponse("VALIDATION", "Invalid JSON body");
+  }
+
+  if (!body.collection || typeof body.collection !== "string") {
+    return errorResponse("VALIDATION", "Missing or invalid collection");
+  }
+  if (body.content !== undefined && typeof body.content !== "string") {
+    return errorResponse("VALIDATION", "content must be a string");
+  }
+  if (body.title !== undefined && typeof body.title !== "string") {
+    return errorResponse("VALIDATION", "title must be a string");
+  }
+  if (body.relPath !== undefined && typeof body.relPath !== "string") {
+    return errorResponse("VALIDATION", "relPath must be a string");
+  }
+  if (body.folderPath !== undefined && typeof body.folderPath !== "string") {
+    return errorResponse("VALIDATION", "folderPath must be a string");
+  }
+  if ("overwrite" in body) {
+    return errorResponse(
+      "VALIDATION",
+      "overwrite is not supported by /api/capture; use collisionPolicy instead"
+    );
+  }
+
+  const collectionName = body.collection.toLowerCase();
+  const collection = ctxHolder.config.collections.find(
+    (candidate) => candidate.name.toLowerCase() === collectionName
+  );
+  if (!collection) {
+    return errorResponse(
+      "NOT_FOUND",
+      `Collection not found: ${body.collection}`,
+      404
+    );
+  }
+
+  let plan: CapturePlan;
+  try {
+    plan = planCapture({
+      input: {
+        ...body,
+        collection: collection.name,
+      },
+      existingRelPaths: await listCollectionRelPaths(store, collection.name),
+      diskRelPaths: await listCaptureDiskRelPaths(collection.path),
+    });
+  } catch (error) {
+    return errorResponse(
+      "VALIDATION",
+      error instanceof Error ? error.message : String(error),
+      409
+    );
+  }
+
+  const fullPath = pathJoin(collection.path, plan.relPath);
+  if (plan.openedExisting) {
+    const existingDoc = await store.getDocument(collection.name, plan.relPath);
+    if (!existingDoc.ok) {
+      return errorResponse("RUNTIME", existingDoc.error.message, 500);
+    }
+    return jsonResponse(
+      buildCaptureReceipt({
+        plan,
+        absPath: fullPath,
+        docid: existingDoc.value?.docid,
+        sync: existingDoc.value
+          ? { status: "completed" }
+          : {
+              status: "skipped",
+              reason: "Existing file is not indexed yet.",
+            },
+      })
+    );
+  }
+
+  try {
+    await mkdir(dirname(fullPath), { recursive: true });
+    ctxHolder.watchService?.suppress(fullPath);
+    await writeCapturePlanFile(plan, fullPath);
+
+    const gnoUri = `gno://${collection.name}/${plan.relPath}`;
+    const jobResult = startJob("sync", async (): Promise<SyncResult> => {
+      const result = await defaultSyncService.syncCollection(
+        collection,
+        store,
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
+      );
+      ctxHolder.scheduler?.notifySyncComplete([plan.relPath]);
+      ctxHolder.eventBus?.emit({
+        type: "document-changed",
+        uri: gnoUri,
+        collection: collection.name,
+        relPath: plan.relPath,
+        origin: "create",
+        changedAt: new Date().toISOString(),
+      });
+      return {
+        collections: [result],
+        totalDurationMs: result.durationMs,
+        totalFilesProcessed: result.filesProcessed,
+        totalFilesAdded: result.filesAdded,
+        totalFilesUpdated: result.filesUpdated,
+        totalFilesErrored: result.filesErrored,
+        totalFilesSkipped: result.filesSkipped,
+      };
+    });
+
+    return jsonResponse(
+      buildCaptureReceipt({
+        plan,
+        absPath: fullPath,
+        sync: jobResult.ok
+          ? {
+              status: "pending",
+              jobId: jobResult.jobId,
+              reason: "Sync job started; poll /api/jobs/:id for status.",
+            }
+          : {
+              status: "skipped",
+              jobId: jobResult.activeJobId,
+              reason: "Sync skipped because another job is running.",
+              error: jobResult.error,
+            },
+      }),
+      202
+    );
+  } catch (error) {
+    return errorResponse(
+      "RUNTIME",
+      `Failed to capture document: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+      500
+    );
+  }
+}
+
 /**
  * POST /api/docs
  * Create a new document in a collection.
@@ -2979,7 +3166,7 @@ export async function handleCreateDoc(
       const result = await defaultSyncService.syncCollection(
         collection,
         store,
-        { runUpdateCmd: false }
+        withContentTypeRules({ runUpdateCmd: false }, ctxHolder.config)
       );
       // Notify scheduler after sync completes (use gnoUri as docid placeholder)
       // The sync will create a proper docid, but we don't have it here yet
@@ -3989,6 +4176,31 @@ export async function routeApi(
     return handlePublishExport(config, store, req);
   }
 
+  if (path === "/api/capture" && req.method === "POST") {
+    const ctxHolder: ContextHolder = {
+      current: {
+        config,
+        store,
+        vectorIndex: null,
+        embedPort: null,
+        expandPort: null,
+        answerPort: null,
+        rerankPort: null,
+        capabilities: {
+          bm25: true,
+          vector: false,
+          hybrid: false,
+          answer: false,
+        },
+      },
+      config,
+      scheduler: null,
+      eventBus: null,
+      watchService: null,
+    };
+    return handleCreateCapture(ctxHolder, store, req);
+  }
+
   if (path === "/api/docs") {
     return handleDocs(store, url);
   }

package/src/serve/server.ts

@@ -23,6 +23,7 @@ import {
   handleCreateFolder,
   handleCreateCollection,
   handleCreateEditableCopy,
+  handleCreateCapture,
   handleCreateDoc,
   handleDeactivateDoc,
   handleDeleteCollection,
@@ -256,6 +257,17 @@ export async function startServer(
             );
           },
         },
+        "/api/capture": {
+          POST: async (req: Request) => {
+            if (!isRequestAllowed(req, port)) {
+              return withSecurityHeaders(forbiddenResponse(), isDev);
+            }
+            return withSecurityHeaders(
+              await handleCreateCapture(ctxHolder, store, req),
+              isDev
+            );
+          },
+        },
         "/api/docs": {
           GET: async (req: Request) => {
             const url = new URL(req.url);

package/src/serve/watch-service.ts

@@ -2,7 +2,7 @@ import { watch, type FSWatcher } from "node:fs";
 import { join, normalize, sep } from "node:path";
 
 import type { Collection } from "../config/types";
-import type { CollectionSyncResult } from "../ingestion";
+import type { CollectionSyncResult, SyncOptions } from "../ingestion";
 import type { SqliteAdapter } from "../store/sqlite/adapter";
 import type { DocumentEvent, DocumentEventBus } from "./doc-events";
 import type { EmbedScheduler } from "./embed-scheduler";
@@ -39,6 +39,7 @@ interface CollectionWatchServiceOptions {
   scheduler: EmbedScheduler | null;
   eventBus?: DocumentEventBus | null;
   callbacks?: CollectionWatchCallbacks;
+  syncOptions?: SyncOptions;
   watchFactory?: typeof watch;
 }
 
@@ -48,6 +49,7 @@ export class CollectionWatchService {
   readonly #scheduler: EmbedScheduler | null;
   readonly #eventBus: DocumentEventBus | null;
   readonly #callbacks: CollectionWatchCallbacks | null;
+  #syncOptions: SyncOptions;
   readonly #watchers = new Map<string, FSWatcher>();
   readonly #pendingByCollection = new Map<string, Set<string>>();
   readonly #timers = new Map<string, ReturnType<typeof setTimeout>>();
@@ -64,6 +66,7 @@ export class CollectionWatchService {
     this.#scheduler = options.scheduler;
     this.#eventBus = options.eventBus ?? null;
     this.#callbacks = options.callbacks ?? null;
+    this.#syncOptions = options.syncOptions ?? {};
     this.#watchFactory = options.watchFactory ?? watch;
   }
 
@@ -71,8 +74,14 @@ export class CollectionWatchService {
     this.updateCollections(this.#collections);
   }
 
-  updateCollections(collections: Collection[]): void {
+  updateCollections(
+    collections: Collection[],
+    syncOptions?: SyncOptions
+  ): void {
     this.#collections = collections;
+    if (syncOptions) {
+      this.#syncOptions = syncOptions;
+    }
     const nextNames = new Set(collections.map((collection) => collection.name));
 
     for (const [collectionName, watcher] of this.#watchers) {
@@ -204,6 +213,7 @@ export class CollectionWatchService {
         collection,
         this.#store,
         {
+          ...this.#syncOptions,
           runUpdateCmd: false,
         }
       );

package/src/store/migrations/009-content-type-rule-fingerprint.ts

@@ -0,0 +1,36 @@
+/**
+ * Migration: content type rule fingerprint.
+ *
+ * @module src/store/migrations/009-content-type-rule-fingerprint
+ */
+
+import type { Migration } from "./runner";
+
+export const migration: Migration = {
+  version: 9,
+  name: "content_type_rule_fingerprint",
+
+  up(db): void {
+    const columns = new Set(
+      db
+        .query<{ name: string }, []>("PRAGMA table_info(documents)")
+        .all()
+        .map((row) => row.name)
+    );
+    if (!columns.has("content_type_rules_fingerprint")) {
+      db.exec(
+        "ALTER TABLE documents ADD COLUMN content_type_rules_fingerprint TEXT"
+      );
+    }
+    db.exec(
+      "CREATE INDEX IF NOT EXISTS idx_documents_content_type_rules_fingerprint ON documents(content_type_rules_fingerprint)"
+    );
+  },
+
+  down(db): void {
+    db.exec(
+      "DROP INDEX IF EXISTS idx_documents_content_type_rules_fingerprint"
+    );
+    // SQLite cannot drop columns without a table rebuild; keep column on rollback.
+  },
+};

package/src/store/migrations/index.ts

@@ -22,6 +22,17 @@ import { migration as m005 } from "./005-graph-indexes";
 import { migration as m006 } from "./006-document-metadata";
 import { migration as m007 } from "./007-document-date-fields";
 import { migration as m008 } from "./008-vector-fingerprints";
+import { migration as m009 } from "./009-content-type-rule-fingerprint";
 
 /** All migrations in order */
-export const migrations = [m001, m002, m003, m004, m005, m006, m007, m008];
+export const migrations = [
+  m001,
+  m002,
+  m003,
+  m004,
+  m005,
+  m006,
+  m007,
+  m008,
+  m009,
+];

package/src/store/sqlite/adapter.ts

@@ -554,9 +554,10 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           collection, rel_path, source_hash, source_mime, source_ext,
           source_size, source_mtime, source_ctime, docid, uri, title, mirror_hash,
           converter_id, converter_version, language_hint, content_type, categories,
-          author, frontmatter_date, date_fields, active, indexed_at, last_error_code,
-          last_error_message, last_error_at, ingest_version, updated_at
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'), ?, ?, ?, ?, datetime('now'))
+          author, frontmatter_date, date_fields, content_type_rules_fingerprint,
+          active, indexed_at, last_error_code, last_error_message, last_error_at,
+          ingest_version, updated_at
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now'), ?, ?, ?, ?, datetime('now'))
         ON CONFLICT(collection, rel_path) DO UPDATE SET
           source_hash = excluded.source_hash,
           source_mime = excluded.source_mime,
@@ -576,6 +577,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           author = excluded.author,
           frontmatter_date = excluded.frontmatter_date,
           date_fields = excluded.date_fields,
+          content_type_rules_fingerprint = excluded.content_type_rules_fingerprint,
           active = 1,
           indexed_at = datetime('now'),
           last_error_code = excluded.last_error_code,
@@ -605,6 +607,7 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           doc.author ?? null,
           doc.frontmatterDate ?? null,
           doc.dateFields ? JSON.stringify(doc.dateFields) : null,
+          doc.contentTypeRulesFingerprint ?? null,
           doc.lastErrorCode ?? null,
           doc.lastErrorMessage ?? null,
           doc.lastErrorCode ? new Date().toISOString() : null,
@@ -1366,7 +1369,9 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           d.source_mtime,
           d.frontmatter_date,
           d.source_size,
-          d.source_hash
+          d.source_hash,
+          d.content_type,
+          d.categories
         FROM fts_matches fm
         JOIN documents d ON d.id = fm.rowid AND d.active = 1
         WHERE 1 = 1
@@ -1392,6 +1397,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
         frontmatter_date: string | null;
         source_size: number | null;
         source_hash: string | null;
+        content_type: string | null;
+        categories: string | null;
       }
 
       const queryParams = [builtQuery.query, ftsLimit, ...params];
@@ -1416,6 +1423,8 @@ export class SqliteAdapter implements StorePort, SqliteDbProvider {
           frontmatterDate: r.frontmatter_date ?? undefined,
           sourceSize: r.source_size ?? undefined,
           sourceHash: r.source_hash ?? undefined,
+          contentType: r.content_type ?? undefined,
+          categories: parseCategoriesJson(r.categories) ?? undefined,
         }))
       );
     } catch (cause) {
@@ -3522,6 +3531,7 @@ interface DbDocumentRow {
   author: string | null;
   frontmatter_date: string | null;
   date_fields: string | null;
+  content_type_rules_fingerprint: string | null;
   indexed_at: string | null;
   active: number;
   ingest_version: number | null;
@@ -3580,19 +3590,23 @@ function mapContextRow(row: DbContextRow): ContextRow {
   };
 }
 
-function mapDocumentRow(row: DbDocumentRow): DocumentRow {
-  let categories: string[] | null = null;
-  if (row.categories) {
-    try {
-      const parsed = JSON.parse(row.categories);
-      if (Array.isArray(parsed)) {
-        categories = parsed.filter((v): v is string => typeof v === "string");
-      }
-    } catch {
-      categories = null;
+function parseCategoriesJson(raw: string | null): string[] | null {
+  if (!raw) {
+    return null;
+  }
+  try {
+    const parsed = JSON.parse(raw);
+    if (Array.isArray(parsed)) {
+      return parsed.filter((v): v is string => typeof v === "string");
     }
+  } catch {
+    return null;
   }
+  return null;
+}
 
+function mapDocumentRow(row: DbDocumentRow): DocumentRow {
+  const categories = parseCategoriesJson(row.categories);
   let dateFields: Record<string, string> | null = null;
   if (row.date_fields) {
     try {
@@ -3638,6 +3652,7 @@ function mapDocumentRow(row: DbDocumentRow): DocumentRow {
     indexedAt: row.indexed_at,
     active: row.active === 1,
     ingestVersion: row.ingest_version,
+    contentTypeRulesFingerprint: row.content_type_rules_fingerprint,
     lastErrorCode: row.last_error_code,
     lastErrorMessage: row.last_error_message,
     lastErrorAt: row.last_error_at,

package/src/store/types.ts

@@ -118,6 +118,8 @@ export interface DocumentRow {
   active: boolean;
   /** Ingest schema version for backfill detection */
   ingestVersion: number | null;
+  /** Fingerprint of normalized content type rules used for derived metadata */
+  contentTypeRulesFingerprint?: string | null;
 
   // Error tracking
   lastErrorCode: string | null;
@@ -264,6 +266,8 @@ export interface DocumentInput {
   lastErrorMessage?: string;
   /** Ingest schema version for backfill detection */
   ingestVersion?: number;
+  /** Fingerprint of normalized content type rules used for derived metadata */
+  contentTypeRulesFingerprint?: string;
 }
 
 /** Result of upserting a document */
@@ -345,6 +349,8 @@ export interface FtsResult {
   frontmatterDate?: string;
   sourceSize?: number;
   sourceHash?: string;
+  contentType?: string;
+  categories?: string[];
 }
 
 // ─────────────────────────────────────────────────────────────────────────────