@gmickel/gno 0.22.6 → 0.25.0

package/src/sdk/documents.ts

@@ -0,0 +1,348 @@
+/**
+ * SDK document retrieval helpers.
+ *
+ * @module src/sdk/documents
+ */
+
+import { minimatch } from "minimatch";
+
+import type { GetResponse } from "../cli/commands/get";
+import type { LsResponse } from "../cli/commands/ls";
+import type {
+  MultiGetDocument,
+  MultiGetResponse,
+  SkippedDoc,
+} from "../cli/commands/multi-get";
+import type { ParsedRef } from "../cli/commands/ref-parser";
+import type { Config } from "../config/types";
+import type { DocumentRow, StorePort } from "../store/types";
+import type {
+  GnoGetOptions,
+  GnoListOptions,
+  GnoMultiGetOptions,
+} from "./types";
+
+import { isGlobPattern, parseRef, splitRefs } from "../cli/commands/ref-parser";
+import { sdkError } from "./errors";
+
+const URI_PREFIX_PATTERN = /^gno:\/\/[^/]+\//;
+const encoder = new TextEncoder();
+
+function lookupDocument(store: StorePort, parsed: ParsedRef) {
+  switch (parsed.type) {
+    case "docid":
+      return store.getDocumentByDocid(parsed.value);
+    case "uri":
+      return store.getDocumentByUri(parsed.value);
+    case "collPath":
+      if (!(parsed.collection && parsed.relPath)) {
+        return Promise.resolve({ ok: true as const, value: null });
+      }
+      return store.getDocument(parsed.collection, parsed.relPath);
+  }
+}
+
+function buildSourceMeta(
+  doc: DocumentRow,
+  config: Config
+): GetResponse["source"] {
+  const coll = config.collections.find((c) => c.name === doc.collection);
+  return {
+    absPath: coll ? `${coll.path}/${doc.relPath}` : undefined,
+    relPath: doc.relPath,
+    mime: doc.sourceMime,
+    ext: doc.sourceExt,
+    sizeBytes: doc.sourceSize,
+    sourceHash: doc.sourceHash,
+  };
+}
+
+function buildConversionMeta(
+  doc: DocumentRow
+): GetResponse["conversion"] | undefined {
+  if (!doc.converterId) {
+    return;
+  }
+  return {
+    converterId: doc.converterId ?? undefined,
+    converterVersion: doc.converterVersion ?? undefined,
+    mirrorHash: doc.mirrorHash ?? undefined,
+  };
+}
+
+export async function getDocumentByRef(
+  store: StorePort,
+  config: Config,
+  ref: string,
+  options: GnoGetOptions = {}
+): Promise<GetResponse> {
+  if (options.from !== undefined && options.from <= 0) {
+    throw sdkError("VALIDATION", "--from must be a positive integer");
+  }
+  if (options.limit !== undefined && options.limit < 0) {
+    throw sdkError("VALIDATION", "limit cannot be negative");
+  }
+
+  const parsed = parseRef(ref);
+  if ("error" in parsed) {
+    throw sdkError("VALIDATION", parsed.error);
+  }
+
+  const docResult = await lookupDocument(store, parsed);
+  if (!docResult.ok) {
+    throw sdkError("STORE", docResult.error.message, {
+      cause: docResult.error.cause,
+    });
+  }
+
+  const doc = docResult.value;
+  if (!doc?.active) {
+    throw sdkError("NOT_FOUND", "Document not found");
+  }
+  if (!doc.mirrorHash) {
+    throw sdkError("RUNTIME", "Mirror content unavailable (conversion error)");
+  }
+
+  const contentResult = await store.getContent(doc.mirrorHash);
+  if (!contentResult.ok || contentResult.value === null) {
+    throw sdkError("RUNTIME", "Mirror content unavailable", {
+      cause: !contentResult.ok ? contentResult.error.cause : undefined,
+    });
+  }
+
+  const lines = contentResult.value.split("\n");
+  const totalLines = lines.length;
+  if (options.limit === 0) {
+    return {
+      docid: doc.docid,
+      uri: doc.uri,
+      title: doc.title ?? undefined,
+      content: "",
+      totalLines,
+      language: doc.languageHint ?? undefined,
+      source: buildSourceMeta(doc, config),
+      conversion: buildConversionMeta(doc),
+    };
+  }
+
+  const startLine = options.from ?? parsed.line ?? 1;
+  const limit = options.limit ?? totalLines;
+  const clampedStart = Math.max(1, Math.min(startLine, totalLines));
+  const clampedEnd = Math.min(clampedStart + limit - 1, totalLines);
+  const selectedLines = lines.slice(clampedStart - 1, clampedEnd);
+  const isPartial = clampedStart > 1 || clampedEnd < totalLines;
+
+  return {
+    docid: doc.docid,
+    uri: doc.uri,
+    title: doc.title ?? undefined,
+    content: selectedLines.join("\n"),
+    totalLines,
+    returnedLines: isPartial
+      ? { start: clampedStart, end: clampedEnd }
+      : undefined,
+    language: doc.languageHint ?? undefined,
+    source: buildSourceMeta(doc, config),
+    conversion: buildConversionMeta(doc),
+  };
+}
+
+function truncateContent(
+  content: string,
+  maxBytes: number
+): { content: string; truncated: boolean } {
+  if (encoder.encode(content).length <= maxBytes) {
+    return { content, truncated: false };
+  }
+
+  const lines = content.split("\n");
+  let accumulated = "";
+  let byteLen = 0;
+  for (const line of lines) {
+    const lineBytes = encoder.encode(`${line}\n`).length;
+    if (byteLen + lineBytes > maxBytes) {
+      return { content: accumulated.trimEnd(), truncated: true };
+    }
+    accumulated += `${line}\n`;
+    byteLen += lineBytes;
+  }
+
+  return { content: accumulated.trimEnd(), truncated: false };
+}
+
+async function expandGlobs(
+  refs: string[],
+  store: StorePort
+): Promise<{ expanded: string[]; invalidRefs: string[] }> {
+  const expanded: string[] = [];
+  const invalidRefs: string[] = [];
+
+  for (const ref of refs) {
+    if (!isGlobPattern(ref)) {
+      expanded.push(ref);
+      continue;
+    }
+
+    const slashIdx = ref.indexOf("/");
+    if (slashIdx === -1) {
+      invalidRefs.push(ref);
+      continue;
+    }
+
+    const collection = ref.slice(0, slashIdx);
+    const pattern = ref.slice(slashIdx + 1);
+    const listResult = await store.listDocuments(collection);
+    if (!listResult.ok) {
+      invalidRefs.push(ref);
+      continue;
+    }
+
+    for (const doc of listResult.value) {
+      if (doc.active && minimatch(doc.relPath, pattern)) {
+        expanded.push(`${collection}/${doc.relPath}`);
+      }
+    }
+  }
+
+  return { expanded, invalidRefs };
+}
+
+export async function multiGetDocuments(
+  store: StorePort,
+  config: Config,
+  refs: string[],
+  options: GnoMultiGetOptions = {}
+): Promise<MultiGetResponse> {
+  const maxBytes = options.maxBytes ?? 10_240;
+  const allRefs = splitRefs(refs);
+  const { expanded, invalidRefs } = await expandGlobs(allRefs, store);
+
+  const documents: MultiGetDocument[] = [];
+  const skipped: SkippedDoc[] = invalidRefs.map((ref) => ({
+    ref,
+    reason: "invalid_ref",
+  }));
+  const seen = new Set<string>();
+
+  for (const ref of expanded) {
+    if (seen.has(ref)) {
+      continue;
+    }
+    seen.add(ref);
+
+    const parsed = parseRef(ref);
+    if ("error" in parsed) {
+      skipped.push({ ref, reason: "invalid_ref" });
+      continue;
+    }
+
+    const docResult = await lookupDocument(store, parsed);
+    if (!docResult.ok || !docResult.value?.active) {
+      skipped.push({ ref, reason: "not_found" });
+      continue;
+    }
+
+    const doc = docResult.value;
+    if (!doc.mirrorHash) {
+      skipped.push({ ref, reason: "conversion_error" });
+      continue;
+    }
+
+    const contentResult = await store.getContent(doc.mirrorHash);
+    if (!contentResult.ok || contentResult.value === null) {
+      skipped.push({ ref, reason: "conversion_error" });
+      continue;
+    }
+
+    const { content, truncated } = truncateContent(
+      contentResult.value,
+      maxBytes
+    );
+    const coll = config.collections.find((c) => c.name === doc.collection);
+    documents.push({
+      docid: doc.docid,
+      uri: doc.uri,
+      title: doc.title ?? undefined,
+      content,
+      truncated: truncated || undefined,
+      totalLines: content.split("\n").length,
+      source: {
+        absPath: coll ? `${coll.path}/${doc.relPath}` : undefined,
+        relPath: doc.relPath,
+        mime: doc.sourceMime,
+        ext: doc.sourceExt,
+      },
+    });
+  }
+
+  return {
+    documents,
+    skipped,
+    meta: {
+      requested: expanded.length + invalidRefs.length,
+      returned: documents.length,
+      skipped: skipped.length,
+      maxBytes,
+    },
+  };
+}
+
+export async function listDocuments(
+  store: StorePort,
+  options: GnoListOptions = {}
+): Promise<LsResponse> {
+  const scope = options.scope;
+  if (scope?.startsWith("gno://")) {
+    if (scope === "gno://") {
+      throw sdkError("VALIDATION", "Invalid scope: missing collection");
+    }
+    if (!URI_PREFIX_PATTERN.test(scope)) {
+      throw sdkError(
+        "VALIDATION",
+        "Invalid scope: missing trailing path (use gno://collection/)"
+      );
+    }
+  }
+
+  const docsResult = !scope
+    ? await store.listDocuments()
+    : scope.startsWith("gno://")
+      ? await store.listDocuments()
+      : await store.listDocuments(scope);
+
+  if (!docsResult.ok) {
+    throw sdkError("STORE", docsResult.error.message, {
+      cause: docsResult.error.cause,
+    });
+  }
+
+  const allActive = docsResult.value
+    .filter((d) => d.active)
+    .filter((d) =>
+      !scope?.startsWith("gno://") ? true : d.uri.startsWith(scope)
+    )
+    .map((d) => ({
+      docid: d.docid,
+      uri: d.uri,
+      title: d.title ?? undefined,
+      source: {
+        relPath: d.relPath,
+        mime: d.sourceMime,
+        ext: d.sourceExt,
+      },
+    }))
+    .sort((a, b) => a.uri.localeCompare(b.uri));
+
+  const offset = options.offset ?? 0;
+  const limit = options.limit ?? 20;
+  const paged = allActive.slice(offset, offset + limit);
+
+  return {
+    documents: paged,
+    meta: {
+      total: allActive.length,
+      returned: paged.length,
+      offset,
+    },
+  };
+}

package/src/sdk/embed.ts

@@ -0,0 +1,287 @@
+/**
+ * SDK embedding helpers.
+ *
+ * @module src/sdk/embed
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type { Config } from "../config/types";
+import type { LlmAdapter } from "../llm/nodeLlamaCpp/adapter";
+import type { EmbeddingPort } from "../llm/types";
+import type { SqliteAdapter } from "../store/sqlite/adapter";
+import type { StoreResult } from "../store/types";
+import type {
+  BacklogItem,
+  VectorIndexPort,
+  VectorStatsPort,
+} from "../store/vector";
+import type { GnoEmbedOptions, GnoEmbedResult } from "./types";
+
+import { embedBacklog } from "../embed";
+import { getActivePreset } from "../llm/registry";
+import { formatDocForEmbedding } from "../pipeline/contextual";
+import { err, ok } from "../store/types";
+import {
+  createVectorIndexPort,
+  createVectorStatsPort,
+  type VectorRow,
+} from "../store/vector";
+import { sdkError } from "./errors";
+
+interface EmbedRuntimeOptions {
+  config: Config;
+  store: SqliteAdapter;
+  llm: LlmAdapter;
+  downloadPolicy?: import("../llm/policy").DownloadPolicy;
+}
+
+function getActiveChunkCount(db: Database): Promise<StoreResult<number>> {
+  try {
+    const result = db
+      .prepare(
+        `
+        SELECT COUNT(*) as count FROM content_chunks c
+        WHERE EXISTS (
+          SELECT 1 FROM documents d
+          WHERE d.mirror_hash = c.mirror_hash AND d.active = 1
+        )
+      `
+      )
+      .get() as { count: number };
+    return Promise.resolve(ok(result.count));
+  } catch (cause) {
+    return Promise.resolve(
+      err(
+        "QUERY_FAILED",
+        cause instanceof Error ? cause.message : "Failed to count chunks",
+        cause
+      )
+    );
+  }
+}
+
+function getActiveChunks(
+  db: Database,
+  limit: number,
+  after?: { mirrorHash: string; seq: number }
+): Promise<StoreResult<BacklogItem[]>> {
+  try {
+    const sql = after
+      ? `
+        SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
+          (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
+          'force' as reason
+        FROM content_chunks c
+        WHERE EXISTS (
+          SELECT 1 FROM documents d
+          WHERE d.mirror_hash = c.mirror_hash AND d.active = 1
+        )
+        AND (c.mirror_hash > ? OR (c.mirror_hash = ? AND c.seq > ?))
+        ORDER BY c.mirror_hash, c.seq
+        LIMIT ?
+      `
+      : `
+        SELECT c.mirror_hash as mirrorHash, c.seq, c.text,
+          (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 LIMIT 1) as title,
+          'force' as reason
+        FROM content_chunks c
+        WHERE EXISTS (
+          SELECT 1 FROM documents d
+          WHERE d.mirror_hash = c.mirror_hash AND d.active = 1
+        )
+        ORDER BY c.mirror_hash, c.seq
+        LIMIT ?
+      `;
+
+    const params = after
+      ? [after.mirrorHash, after.mirrorHash, after.seq, limit]
+      : [limit];
+    const result = db.prepare(sql).all(...params) as BacklogItem[];
+    return Promise.resolve(ok(result));
+  } catch (cause) {
+    return Promise.resolve(
+      err(
+        "QUERY_FAILED",
+        cause instanceof Error ? cause.message : "Failed to get chunks",
+        cause
+      )
+    );
+  }
+}
+
+async function forceEmbedAll(
+  db: Database,
+  embedPort: EmbeddingPort,
+  vectorIndex: VectorIndexPort,
+  modelUri: string,
+  batchSize: number
+): Promise<{ embedded: number; errors: number }> {
+  let embedded = 0;
+  let errors = 0;
+  let cursor: { mirrorHash: string; seq: number } | undefined;
+
+  while (true) {
+    const batchResult = await getActiveChunks(db, batchSize, cursor);
+    if (!batchResult.ok) {
+      throw sdkError("STORE", batchResult.error.message, {
+        cause: batchResult.error.cause,
+      });
+    }
+
+    const batch = batchResult.value;
+    if (batch.length === 0) {
+      break;
+    }
+
+    const lastItem = batch.at(-1);
+    if (lastItem) {
+      cursor = { mirrorHash: lastItem.mirrorHash, seq: lastItem.seq };
+    }
+
+    const embedResult = await embedPort.embedBatch(
+      batch.map((item) =>
+        formatDocForEmbedding(item.text, item.title ?? undefined)
+      )
+    );
+    if (!embedResult.ok || embedResult.value.length !== batch.length) {
+      errors += batch.length;
+      continue;
+    }
+
+    const vectors: VectorRow[] = batch.map((item, idx) => ({
+      mirrorHash: item.mirrorHash,
+      seq: item.seq,
+      model: modelUri,
+      embedding: new Float32Array(embedResult.value[idx] as number[]),
+    }));
+    const storeResult = await vectorIndex.upsertVectors(vectors);
+    if (!storeResult.ok) {
+      errors += batch.length;
+      continue;
+    }
+
+    embedded += batch.length;
+  }
+
+  if (vectorIndex.vecDirty) {
+    const syncResult = await vectorIndex.syncVecIndex();
+    if (syncResult.ok) {
+      vectorIndex.vecDirty = false;
+    }
+  }
+
+  return { embedded, errors };
+}
+
+async function checkVecAvailable(db: Database): Promise<boolean> {
+  try {
+    const sqliteVec = await import("sqlite-vec");
+    sqliteVec.load(db);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function runEmbed(
+  runtime: EmbedRuntimeOptions,
+  options: GnoEmbedOptions = {}
+): Promise<GnoEmbedResult> {
+  const batchSize = options.batchSize ?? 32;
+  const force = options.force ?? false;
+  const dryRun = options.dryRun ?? false;
+  const preset = getActivePreset(runtime.config);
+  const modelUri = options.model ?? preset.embed;
+  const db = runtime.store.getRawDb();
+  const stats: VectorStatsPort = createVectorStatsPort(db);
+
+  const backlogResult = force
+    ? await getActiveChunkCount(db)
+    : await stats.countBacklog(modelUri);
+  if (!backlogResult.ok) {
+    throw sdkError("STORE", backlogResult.error.message, {
+      cause: backlogResult.error.cause,
+    });
+  }
+
+  const totalToEmbed = backlogResult.value;
+  if (totalToEmbed === 0 || dryRun) {
+    return {
+      embedded: totalToEmbed,
+      errors: 0,
+      duration: 0,
+      model: modelUri,
+      searchAvailable: await checkVecAvailable(db),
+    };
+  }
+
+  const embedResult = await runtime.llm.createEmbeddingPort(modelUri, {
+    policy: runtime.downloadPolicy,
+  });
+  if (!embedResult.ok) {
+    throw sdkError("MODEL", embedResult.error.message, {
+      cause: embedResult.error.cause,
+    });
+  }
+
+  const embedPort = embedResult.value;
+  try {
+    const probeResult = await embedPort.embed("dimension probe");
+    if (!probeResult.ok) {
+      throw sdkError("MODEL", probeResult.error.message, {
+        cause: probeResult.error.cause,
+      });
+    }
+
+    const vectorResult = await createVectorIndexPort(db, {
+      model: modelUri,
+      dimensions: probeResult.value.length,
+    });
+    if (!vectorResult.ok) {
+      throw sdkError("STORE", vectorResult.error.message, {
+        cause: vectorResult.error.cause,
+      });
+    }
+
+    const vectorIndex = vectorResult.value;
+    const startedAt = Date.now();
+    let result: { embedded: number; errors: number };
+    if (force) {
+      result = await forceEmbedAll(
+        db,
+        embedPort,
+        vectorIndex,
+        modelUri,
+        batchSize
+      );
+    } else {
+      const processed = await embedBacklog({
+        statsPort: stats,
+        embedPort,
+        vectorIndex,
+        modelUri,
+        batchSize,
+      });
+      if (!processed.ok) {
+        throw sdkError("STORE", processed.error.message, {
+          cause: processed.error.cause,
+        });
+      }
+      result = {
+        embedded: processed.value.embedded,
+        errors: processed.value.errors,
+      };
+    }
+
+    return {
+      embedded: result.embedded,
+      errors: result.errors,
+      duration: (Date.now() - startedAt) / 1000,
+      model: modelUri,
+      searchAvailable: vectorIndex.searchAvailable,
+    };
+  } finally {
+    await embedPort.dispose();
+  }
+}

package/src/sdk/errors.ts

@@ -0,0 +1,42 @@
+/**
+ * SDK error types.
+ *
+ * @module src/sdk/errors
+ */
+
+export type GnoSdkErrorCode =
+  | "CONFIG"
+  | "VALIDATION"
+  | "RUNTIME"
+  | "STORE"
+  | "MODEL"
+  | "NOT_FOUND";
+
+export interface GnoSdkErrorOptions {
+  cause?: unknown;
+  details?: Record<string, unknown>;
+}
+
+export class GnoSdkError extends Error {
+  readonly code: GnoSdkErrorCode;
+  readonly details?: Record<string, unknown>;
+
+  constructor(
+    code: GnoSdkErrorCode,
+    message: string,
+    options: GnoSdkErrorOptions = {}
+  ) {
+    super(message, { cause: options.cause });
+    this.name = "GnoSdkError";
+    this.code = code;
+    this.details = options.details;
+  }
+}
+
+export function sdkError(
+  code: GnoSdkErrorCode,
+  message: string,
+  options: GnoSdkErrorOptions = {}
+): GnoSdkError {
+  return new GnoSdkError(code, message, options);
+}