@digitalvibes/ai-knowledge-db 0.1.0

package/dist/index.js

@@ -0,0 +1,292 @@
+// src/client.ts
+import pg from "pg";
+
+// src/config.ts
+var MODEL_DIMENSIONS = {
+  "text-embedding-3-small": 1536,
+  "text-embedding-3-large": 3072,
+  "text-embedding-ada-002": 1536
+};
+var env = (key) => {
+  const v = process.env[key];
+  return v && v.trim() !== "" ? v.trim() : void 0;
+};
+function resolveConfig(config = {}) {
+  const connectionString = config.connectionString ?? env("KNOWLEDGE_DB_URL");
+  if (!connectionString) {
+    throw new Error(
+      "[ai-knowledge-db] Missing connection string. Set KNOWLEDGE_DB_URL in your project's .env or pass { connectionString } to createKnowledgeDB()."
+    );
+  }
+  const openaiApiKey = config.openaiApiKey ?? env("OPENAI_API_KEY");
+  if (!openaiApiKey) {
+    throw new Error(
+      "[ai-knowledge-db] Missing OpenAI key. Set OPENAI_API_KEY in your project's .env or pass { openaiApiKey } to createKnowledgeDB()."
+    );
+  }
+  const embeddingModel = config.embeddingModel ?? env("KNOWLEDGE_EMBED_MODEL") ?? "text-embedding-3-small";
+  const embeddingDimensions = MODEL_DIMENSIONS[embeddingModel] ?? 1536;
+  return {
+    connectionString,
+    openaiApiKey,
+    embeddingModel,
+    embeddingDimensions,
+    clientId: config.clientId ?? env("KNOWLEDGE_CLIENT_ID"),
+    projectId: config.projectId ?? env("KNOWLEDGE_PROJECT_ID")
+  };
+}
+
+// src/embeddings.ts
+import OpenAI from "openai";
+var Embedder = class {
+  client;
+  model;
+  constructor(config) {
+    this.client = new OpenAI({ apiKey: config.openaiApiKey });
+    this.model = config.embeddingModel;
+  }
+  /** Embed a batch of strings in one API call. */
+  async embed(texts) {
+    if (texts.length === 0) return [];
+    const res = await this.client.embeddings.create({
+      model: this.model,
+      input: texts
+    });
+    return res.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
+  }
+  async embedOne(text) {
+    const [vec] = await this.embed([text]);
+    return vec;
+  }
+};
+function chunkText(text, opts = {}) {
+  const maxChars = opts.maxChars ?? 1200;
+  const overlap = opts.overlap ?? 150;
+  const clean = text.replace(/\r\n/g, "\n").trim();
+  if (clean.length <= maxChars) return clean ? [clean] : [];
+  const units = clean.split(/\n{2,}/).flatMap((p) => splitLongUnit(p, maxChars));
+  const chunks = [];
+  let current = "";
+  for (const unit of units) {
+    if (current && current.length + unit.length + 2 > maxChars) {
+      chunks.push(current.trim());
+      current = overlap > 0 ? current.slice(-overlap) + "\n\n" + unit : unit;
+    } else {
+      current = current ? current + "\n\n" + unit : unit;
+    }
+  }
+  if (current.trim()) chunks.push(current.trim());
+  return chunks;
+}
+function splitLongUnit(unit, maxChars) {
+  if (unit.length <= maxChars) return [unit];
+  const sentences = unit.match(/[^.!?\n]+[.!?]?\s*/g) ?? [unit];
+  const out = [];
+  let buf = "";
+  for (const s of sentences) {
+    if (s.length > maxChars) {
+      if (buf) {
+        out.push(buf);
+        buf = "";
+      }
+      for (let i = 0; i < s.length; i += maxChars) out.push(s.slice(i, i + maxChars));
+    } else if (buf.length + s.length > maxChars) {
+      out.push(buf);
+      buf = s;
+    } else {
+      buf += s;
+    }
+  }
+  if (buf) out.push(buf);
+  return out;
+}
+
+// src/schema.ts
+var SCHEMA_SQL = `-- AI Knowledge DB schema (Postgres + pgvector)
+-- Run once against your Hetzner/EasyPanel Postgres instance, or via kb.init().
+-- If you switch to text-embedding-3-large, change 1536 -> 3072 and re-index.
+
+create extension if not exists vector;
+create extension if not exists "pgcrypto"; -- for gen_random_uuid()
+
+create table if not exists knowledge (
+  id          uuid primary key default gen_random_uuid(),
+  scope       text not null check (scope in ('global', 'client', 'project')),
+  client_id   text,
+  project_id  text,
+  source      text,
+  content     text not null,
+  embedding   vector(1536) not null,
+  metadata    jsonb not null default '{}',
+  created_at  timestamptz not null default now()
+);
+
+create index if not exists knowledge_client_idx   on knowledge (client_id);
+create index if not exists knowledge_project_idx  on knowledge (project_id);
+create index if not exists knowledge_scope_idx    on knowledge (scope);
+create index if not exists knowledge_metadata_idx on knowledge using gin (metadata);
+
+create index if not exists knowledge_embedding_idx
+  on knowledge using hnsw (embedding vector_cosine_ops);
+`;
+
+// src/client.ts
+var { Pool } = pg;
+var KnowledgeDB = class {
+  pool;
+  embedder;
+  config;
+  constructor(config = {}) {
+    this.config = resolveConfig(config);
+    this.pool = new Pool({ connectionString: this.config.connectionString });
+    this.embedder = new Embedder(this.config);
+  }
+  /** Create the extension, table, and indexes if they don't exist. Safe to call repeatedly. */
+  async init() {
+    await this.pool.query(SCHEMA_SQL);
+  }
+  /**
+   * Add knowledge. The content is chunked, embedded, and stored. Returns the
+   * ids of the stored rows (one per chunk). Scope/client/project fall back to
+   * the env-configured defaults.
+   */
+  async add(input) {
+    const clientId = input.clientId ?? this.config.clientId ?? null;
+    const projectId = input.projectId ?? this.config.projectId ?? null;
+    const scope = input.scope ?? defaultScope(clientId, projectId);
+    const source = input.source ?? null;
+    const metadata = input.metadata ?? {};
+    const chunks = chunkText(input.content, input.chunking);
+    if (chunks.length === 0) return [];
+    const vectors = await this.embedder.embed(chunks);
+    const ids = [];
+    const client = await this.pool.connect();
+    try {
+      await client.query("begin");
+      for (let i = 0; i < chunks.length; i++) {
+        const chunkMeta = chunks.length > 1 ? { ...metadata, chunk: i, chunks: chunks.length } : metadata;
+        const { rows } = await client.query(
+          `insert into knowledge (scope, client_id, project_id, source, content, embedding, metadata)
+           values ($1, $2, $3, $4, $5, $6, $7) returning id`,
+          [scope, clientId, projectId, source, chunks[i], toVector(vectors[i]), chunkMeta]
+        );
+        ids.push(rows[0].id);
+      }
+      await client.query("commit");
+    } catch (err) {
+      await client.query("rollback");
+      throw err;
+    } finally {
+      client.release();
+    }
+    return ids;
+  }
+  /**
+   * Replace knowledge from a given source. Deletes existing rows that match the
+   * same scope + ids + source, then re-adds. Use this for idempotent re-ingest
+   * of a file or URL so you don't accumulate duplicates.
+   */
+  async upsertSource(input) {
+    const clientId = input.clientId ?? this.config.clientId ?? null;
+    const projectId = input.projectId ?? this.config.projectId ?? null;
+    const scope = input.scope ?? defaultScope(clientId, projectId);
+    await this.delete({ scope, clientId: clientId ?? void 0, projectId: projectId ?? void 0, source: input.source });
+    return this.add(input);
+  }
+  /** Semantic search, scoped to client / project / global knowledge. */
+  async search(query, opts = {}) {
+    const clientId = opts.clientId ?? this.config.clientId;
+    const projectId = opts.projectId ?? this.config.projectId;
+    const includeClient = opts.includeClientKnowledge ?? true;
+    const includeGlobal = opts.includeGlobal ?? true;
+    const limit = opts.limit ?? 8;
+    const minScore = opts.minScore ?? 0;
+    const queryVec = toVector(await this.embedder.embedOne(query));
+    const orClauses = [];
+    const params = [queryVec];
+    const p = (v) => `$${params.push(v)}`;
+    if (projectId && allows(opts.scopes, "project")) {
+      orClauses.push(`(scope = 'project' and project_id = ${p(projectId)})`);
+    }
+    if (clientId && includeClient && allows(opts.scopes, "client")) {
+      orClauses.push(`(scope = 'client' and client_id = ${p(clientId)})`);
+    }
+    if (includeGlobal && allows(opts.scopes, "global")) {
+      orClauses.push(`scope = 'global'`);
+    }
+    const scopeClause = orClauses.length ? `(${orClauses.join(" or ")})` : `scope = 'global'`;
+    const where = [scopeClause];
+    if (opts.metadata) {
+      where.push(`metadata @> ${p(JSON.stringify(opts.metadata))}::jsonb`);
+    }
+    const { rows } = await this.pool.query(
+      `select id, scope, client_id, project_id, source, content, metadata, created_at,
+              1 - (embedding <=> $1) as score
+       from knowledge
+       where ${where.join(" and ")}
+       order by embedding <=> $1
+       limit ${p(limit)}`,
+      params
+    );
+    return rows.map(rowToResult).filter((r) => r.score >= minScore);
+  }
+  /** Delete rows matching a filter. Returns the number deleted. */
+  async delete(filter) {
+    const where = [];
+    const params = [];
+    const p = (v) => `$${params.push(v)}`;
+    if (filter.id) where.push(`id = ${p(filter.id)}`);
+    if (filter.scope) where.push(`scope = ${p(filter.scope)}`);
+    if (filter.clientId) where.push(`client_id = ${p(filter.clientId)}`);
+    if (filter.projectId) where.push(`project_id = ${p(filter.projectId)}`);
+    if (filter.source) where.push(`source = ${p(filter.source)}`);
+    if (where.length === 0) {
+      throw new Error("[ai-knowledge-db] delete() requires at least one filter to avoid wiping the table.");
+    }
+    const { rowCount } = await this.pool.query(
+      `delete from knowledge where ${where.join(" and ")}`,
+      params
+    );
+    return rowCount ?? 0;
+  }
+  /** Close the connection pool. Call on shutdown. */
+  async close() {
+    await this.pool.end();
+  }
+};
+function createKnowledgeDB(config) {
+  return new KnowledgeDB(config);
+}
+function defaultScope(clientId, projectId) {
+  if (projectId) return "project";
+  if (clientId) return "client";
+  return "global";
+}
+function allows(scopes, scope) {
+  return !scopes || scopes.includes(scope);
+}
+function toVector(vec) {
+  return `[${vec.join(",")}]`;
+}
+function rowToResult(row) {
+  return { ...rowToRecord(row), score: Number(row.score) };
+}
+function rowToRecord(row) {
+  return {
+    id: row.id,
+    scope: row.scope,
+    clientId: row.client_id,
+    projectId: row.project_id,
+    source: row.source,
+    content: row.content,
+    metadata: row.metadata ?? {},
+    createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : row.created_at
+  };
+}
+export {
+  KnowledgeDB,
+  chunkText,
+  createKnowledgeDB,
+  resolveConfig
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/client.ts","../src/config.ts","../src/embeddings.ts","../src/schema.ts"],"sourcesContent":["import pg from \"pg\";\nimport { resolveConfig, type KnowledgeConfig, type ResolvedConfig } from \"./config.js\";\nimport { Embedder, chunkText } from \"./embeddings.js\";\nimport { SCHEMA_SQL } from \"./schema.js\";\nimport type {\n  AddInput,\n  DeleteFilter,\n  KnowledgeRecord,\n  Scope,\n  SearchOptions,\n  SearchResult,\n} from \"./types.js\";\n\nconst { Pool } = pg;\n\nexport class KnowledgeDB {\n  private pool: pg.Pool;\n  private embedder: Embedder;\n  readonly config: ResolvedConfig;\n\n  constructor(config: KnowledgeConfig = {}) {\n    this.config = resolveConfig(config);\n    this.pool = new Pool({ connectionString: this.config.connectionString });\n    this.embedder = new Embedder(this.config);\n  }\n\n  /** Create the extension, table, and indexes if they don't exist. Safe to call repeatedly. */\n  async init(): Promise<void> {\n    await this.pool.query(SCHEMA_SQL);\n  }\n\n  /**\n   * Add knowledge. The content is chunked, embedded, and stored. Returns the\n   * ids of the stored rows (one per chunk). Scope/client/project fall back to\n   * the env-configured defaults.\n   */\n  async add(input: AddInput): Promise<string[]> {\n    const clientId = input.clientId ?? this.config.clientId ?? null;\n    const projectId = input.projectId ?? this.config.projectId ?? null;\n    const scope = input.scope ?? defaultScope(clientId, projectId);\n    const source = input.source ?? null;\n    const metadata = input.metadata ?? {};\n\n    const chunks = chunkText(input.content, input.chunking);\n    if (chunks.length === 0) return [];\n\n    const vectors = await this.embedder.embed(chunks);\n    const ids: string[] = [];\n\n    const client = await this.pool.connect();\n    try {\n      await client.query(\"begin\");\n      for (let i = 0; i < chunks.length; i++) {\n        const chunkMeta =\n          chunks.length > 1\n            ? { ...metadata, chunk: i, chunks: chunks.length }\n            : metadata;\n        const { rows } = await client.query(\n          `insert into knowledge (scope, client_id, project_id, source, content, embedding, metadata)\n           values ($1, $2, $3, $4, $5, $6, $7) returning id`,\n          [scope, clientId, projectId, source, chunks[i], toVector(vectors[i]), chunkMeta],\n        );\n        ids.push(rows[0].id);\n      }\n      await client.query(\"commit\");\n    } catch (err) {\n      await client.query(\"rollback\");\n      throw err;\n    } finally {\n      client.release();\n    }\n    return ids;\n  }\n\n  /**\n   * Replace knowledge from a given source. Deletes existing rows that match the\n   * same scope + ids + source, then re-adds. Use this for idempotent re-ingest\n   * of a file or URL so you don't accumulate duplicates.\n   */\n  async upsertSource(input: AddInput & { source: string }): Promise<string[]> {\n    const clientId = input.clientId ?? this.config.clientId ?? null;\n    const projectId = input.projectId ?? this.config.projectId ?? null;\n    const scope = input.scope ?? defaultScope(clientId, projectId);\n    await this.delete({ scope, clientId: clientId ?? undefined, projectId: projectId ?? undefined, source: input.source });\n    return this.add(input);\n  }\n\n  /** Semantic search, scoped to client / project / global knowledge. */\n  async search(query: string, opts: SearchOptions = {}): Promise<SearchResult[]> {\n    const clientId = opts.clientId ?? this.config.clientId;\n    const projectId = opts.projectId ?? this.config.projectId;\n    const includeClient = opts.includeClientKnowledge ?? true;\n    const includeGlobal = opts.includeGlobal ?? true;\n    const limit = opts.limit ?? 8;\n    const minScore = opts.minScore ?? 0;\n\n    const queryVec = toVector(await this.embedder.embedOne(query));\n\n    // Build a scope clause: project rows, optionally the client's shared rows,\n    // optionally global rows — restricted to the requested scopes if given.\n    const orClauses: string[] = [];\n    const params: unknown[] = [queryVec];\n    const p = (v: unknown) => `$${params.push(v)}`;\n\n    if (projectId && allows(opts.scopes, \"project\")) {\n      orClauses.push(`(scope = 'project' and project_id = ${p(projectId)})`);\n    }\n    if (clientId && includeClient && allows(opts.scopes, \"client\")) {\n      orClauses.push(`(scope = 'client' and client_id = ${p(clientId)})`);\n    }\n    if (includeGlobal && allows(opts.scopes, \"global\")) {\n      orClauses.push(`scope = 'global'`);\n    }\n    // If nothing matched (e.g. no ids at all), fall back to global-only.\n    const scopeClause = orClauses.length ? `(${orClauses.join(\" or \")})` : `scope = 'global'`;\n\n    const where: string[] = [scopeClause];\n    if (opts.metadata) {\n      where.push(`metadata @> ${p(JSON.stringify(opts.metadata))}::jsonb`);\n    }\n\n    const { rows } = await this.pool.query(\n      `select id, scope, client_id, project_id, source, content, metadata, created_at,\n              1 - (embedding <=> $1) as score\n       from knowledge\n       where ${where.join(\" and \")}\n       order by embedding <=> $1\n       limit ${p(limit)}`,\n      params,\n    );\n\n    return rows\n      .map(rowToResult)\n      .filter((r) => r.score >= minScore);\n  }\n\n  /** Delete rows matching a filter. Returns the number deleted. */\n  async delete(filter: DeleteFilter): Promise<number> {\n    const where: string[] = [];\n    const params: unknown[] = [];\n    const p = (v: unknown) => `$${params.push(v)}`;\n    if (filter.id) where.push(`id = ${p(filter.id)}`);\n    if (filter.scope) where.push(`scope = ${p(filter.scope)}`);\n    if (filter.clientId) where.push(`client_id = ${p(filter.clientId)}`);\n    if (filter.projectId) where.push(`project_id = ${p(filter.projectId)}`);\n    if (filter.source) where.push(`source = ${p(filter.source)}`);\n    if (where.length === 0) {\n      throw new Error(\"[ai-knowledge-db] delete() requires at least one filter to avoid wiping the table.\");\n    }\n    const { rowCount } = await this.pool.query(\n      `delete from knowledge where ${where.join(\" and \")}`,\n      params,\n    );\n    return rowCount ?? 0;\n  }\n\n  /** Close the connection pool. Call on shutdown. */\n  async close(): Promise<void> {\n    await this.pool.end();\n  }\n}\n\nexport function createKnowledgeDB(config?: KnowledgeConfig): KnowledgeDB {\n  return new KnowledgeDB(config);\n}\n\nfunction defaultScope(clientId: string | null, projectId: string | null): Scope {\n  if (projectId) return \"project\";\n  if (clientId) return \"client\";\n  return \"global\";\n}\n\nfunction allows(scopes: Scope[] | undefined, scope: Scope): boolean {\n  return !scopes || scopes.includes(scope);\n}\n\n/** pgvector accepts a vector literal like '[0.1,0.2,...]'. */\nfunction toVector(vec: number[]): string {\n  return `[${vec.join(\",\")}]`;\n}\n\nfunction rowToResult(row: any): SearchResult {\n  return { ...rowToRecord(row), score: Number(row.score) };\n}\n\nfunction rowToRecord(row: any): KnowledgeRecord {\n  return {\n    id: row.id,\n    scope: row.scope,\n    clientId: row.client_id,\n    projectId: row.project_id,\n    source: row.source,\n    content: row.content,\n    metadata: row.metadata ?? {},\n    createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : row.created_at,\n  };\n}\n","/**\n * All sensitive / per-deployment values live in the *consuming* project's\n * environment — never in this package. A website repo that installs\n * `@dibe/ai-knowledge-db` sets these in its own `.env`:\n *\n *   KNOWLEDGE_DB_URL=postgres://user:pass@host:5432/knowledge   (Hetzner/EasyPanel)\n *   OPENAI_API_KEY=sk-...\n *   KNOWLEDGE_CLIENT_ID=acme-corp          # default client for this repo\n *   KNOWLEDGE_PROJECT_ID=acme-website-2026 # default project for this repo\n *\n * Anything passed explicitly to createKnowledgeDB() overrides the env value,\n * but env is the intended default so callers usually pass nothing.\n */\n\nexport interface KnowledgeConfig {\n  /** Postgres connection string. Defaults to env KNOWLEDGE_DB_URL. */\n  connectionString?: string;\n  /** OpenAI API key. Defaults to env OPENAI_API_KEY. */\n  openaiApiKey?: string;\n  /** Embedding model. Defaults to env KNOWLEDGE_EMBED_MODEL or text-embedding-3-small. */\n  embeddingModel?: string;\n  /** Default client scope for this repo. Defaults to env KNOWLEDGE_CLIENT_ID. */\n  clientId?: string;\n  /** Default project scope for this repo. Defaults to env KNOWLEDGE_PROJECT_ID. */\n  projectId?: string;\n}\n\nexport interface ResolvedConfig {\n  connectionString: string;\n  openaiApiKey: string;\n  embeddingModel: string;\n  embeddingDimensions: number;\n  clientId?: string;\n  projectId?: string;\n}\n\n/** text-embedding-3-small → 1536, text-embedding-3-large → 3072. */\nconst MODEL_DIMENSIONS: Record<string, number> = {\n  \"text-embedding-3-small\": 1536,\n  \"text-embedding-3-large\": 3072,\n  \"text-embedding-ada-002\": 1536,\n};\n\nconst env = (key: string): string | undefined => {\n  const v = process.env[key];\n  return v && v.trim() !== \"\" ? v.trim() : undefined;\n};\n\nexport function resolveConfig(config: KnowledgeConfig = {}): ResolvedConfig {\n  const connectionString = config.connectionString ?? env(\"KNOWLEDGE_DB_URL\");\n  if (!connectionString) {\n    throw new Error(\n      \"[ai-knowledge-db] Missing connection string. Set KNOWLEDGE_DB_URL in your project's .env \" +\n        \"or pass { connectionString } to createKnowledgeDB().\",\n    );\n  }\n\n  const openaiApiKey = config.openaiApiKey ?? env(\"OPENAI_API_KEY\");\n  if (!openaiApiKey) {\n    throw new Error(\n      \"[ai-knowledge-db] Missing OpenAI key. Set OPENAI_API_KEY in your project's .env \" +\n        \"or pass { openaiApiKey } to createKnowledgeDB().\",\n    );\n  }\n\n  const embeddingModel =\n    config.embeddingModel ?? env(\"KNOWLEDGE_EMBED_MODEL\") ?? \"text-embedding-3-small\";\n  const embeddingDimensions = MODEL_DIMENSIONS[embeddingModel] ?? 1536;\n\n  return {\n    connectionString,\n    openaiApiKey,\n    embeddingModel,\n    embeddingDimensions,\n    clientId: config.clientId ?? env(\"KNOWLEDGE_CLIENT_ID\"),\n    projectId: config.projectId ?? env(\"KNOWLEDGE_PROJECT_ID\"),\n  };\n}\n","import OpenAI from \"openai\";\nimport type { ResolvedConfig } from \"./config.js\";\nimport type { ChunkOptions } from \"./types.js\";\n\nexport class Embedder {\n  private client: OpenAI;\n  private model: string;\n\n  constructor(config: ResolvedConfig) {\n    this.client = new OpenAI({ apiKey: config.openaiApiKey });\n    this.model = config.embeddingModel;\n  }\n\n  /** Embed a batch of strings in one API call. */\n  async embed(texts: string[]): Promise<number[][]> {\n    if (texts.length === 0) return [];\n    const res = await this.client.embeddings.create({\n      model: this.model,\n      input: texts,\n    });\n    // OpenAI preserves input order in the response.\n    return res.data\n      .sort((a, b) => a.index - b.index)\n      .map((d) => d.embedding as number[]);\n  }\n\n  async embedOne(text: string): Promise<number[]> {\n    const [vec] = await this.embed([text]);\n    return vec;\n  }\n}\n\n/**\n * Split text into overlapping chunks. Prefers paragraph boundaries, then\n * sentence boundaries, falling back to hard character cuts for very long runs.\n */\nexport function chunkText(text: string, opts: ChunkOptions = {}): string[] {\n  const maxChars = opts.maxChars ?? 1200;\n  const overlap = opts.overlap ?? 150;\n  const clean = text.replace(/\\r\\n/g, \"\\n\").trim();\n  if (clean.length <= maxChars) return clean ? [clean] : [];\n\n  // Split into paragraph-ish units first.\n  const units = clean.split(/\\n{2,}/).flatMap((p) => splitLongUnit(p, maxChars));\n\n  const chunks: string[] = [];\n  let current = \"\";\n  for (const unit of units) {\n    if (current && current.length + unit.length + 2 > maxChars) {\n      chunks.push(current.trim());\n      // carry overlap from the tail of the previous chunk\n      current = overlap > 0 ? current.slice(-overlap) + \"\\n\\n\" + unit : unit;\n    } else {\n      current = current ? current + \"\\n\\n\" + unit : unit;\n    }\n  }\n  if (current.trim()) chunks.push(current.trim());\n  return chunks;\n}\n\n/** Break a single oversized paragraph on sentence, then hard, boundaries. */\nfunction splitLongUnit(unit: string, maxChars: number): string[] {\n  if (unit.length <= maxChars) return [unit];\n  const sentences = unit.match(/[^.!?\\n]+[.!?]?\\s*/g) ?? [unit];\n  const out: string[] = [];\n  let buf = \"\";\n  for (const s of sentences) {\n    if (s.length > maxChars) {\n      if (buf) {\n        out.push(buf);\n        buf = \"\";\n      }\n      for (let i = 0; i < s.length; i += maxChars) out.push(s.slice(i, i + maxChars));\n    } else if (buf.length + s.length > maxChars) {\n      out.push(buf);\n      buf = s;\n    } else {\n      buf += s;\n    }\n  }\n  if (buf) out.push(buf);\n  return out;\n}\n","/**\n * Canonical schema (Postgres + pgvector), kept as a string so the library never\n * has to read from disk — works identically in the ESM and CJS builds. The\n * build also writes this out to dist/schema.sql for the `./schema.sql` export\n * and for running by hand. Vector size matches text-embedding-3-small (1536).\n */\nexport const SCHEMA_SQL = `-- AI Knowledge DB schema (Postgres + pgvector)\n-- Run once against your Hetzner/EasyPanel Postgres instance, or via kb.init().\n-- If you switch to text-embedding-3-large, change 1536 -> 3072 and re-index.\n\ncreate extension if not exists vector;\ncreate extension if not exists \"pgcrypto\"; -- for gen_random_uuid()\n\ncreate table if not exists knowledge (\n  id          uuid primary key default gen_random_uuid(),\n  scope       text not null check (scope in ('global', 'client', 'project')),\n  client_id   text,\n  project_id  text,\n  source      text,\n  content     text not null,\n  embedding   vector(1536) not null,\n  metadata    jsonb not null default '{}',\n  created_at  timestamptz not null default now()\n);\n\ncreate index if not exists knowledge_client_idx   on knowledge (client_id);\ncreate index if not exists knowledge_project_idx  on knowledge (project_id);\ncreate index if not exists knowledge_scope_idx    on knowledge (scope);\ncreate index if not exists knowledge_metadata_idx on knowledge using gin (metadata);\n\ncreate index if not exists knowledge_embedding_idx\n  on knowledge using hnsw (embedding vector_cosine_ops);\n`;\n"],"mappings":";AAAA,OAAO,QAAQ;;;ACqCf,IAAM,mBAA2C;AAAA,EAC/C,0BAA0B;AAAA,EAC1B,0BAA0B;AAAA,EAC1B,0BAA0B;AAC5B;AAEA,IAAM,MAAM,CAAC,QAAoC;AAC/C,QAAM,IAAI,QAAQ,IAAI,GAAG;AACzB,SAAO,KAAK,EAAE,KAAK,MAAM,KAAK,EAAE,KAAK,IAAI;AAC3C;AAEO,SAAS,cAAc,SAA0B,CAAC,GAAmB;AAC1E,QAAM,mBAAmB,OAAO,oBAAoB,IAAI,kBAAkB;AAC1E,MAAI,CAAC,kBAAkB;AACrB,UAAM,IAAI;AAAA,MACR;AAAA,IAEF;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,gBAAgB,IAAI,gBAAgB;AAChE,MAAI,CAAC,cAAc;AACjB,UAAM,IAAI;AAAA,MACR;AAAA,IAEF;AAAA,EACF;AAEA,QAAM,iBACJ,OAAO,kBAAkB,IAAI,uBAAuB,KAAK;AAC3D,QAAM,sBAAsB,iBAAiB,cAAc,KAAK;AAEhE,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,OAAO,YAAY,IAAI,qBAAqB;AAAA,IACtD,WAAW,OAAO,aAAa,IAAI,sBAAsB;AAAA,EAC3D;AACF;;;AC7EA,OAAO,YAAY;AAIZ,IAAM,WAAN,MAAe;AAAA,EACZ;AAAA,EACA;AAAA,EAER,YAAY,QAAwB;AAClC,SAAK,SAAS,IAAI,OAAO,EAAE,QAAQ,OAAO,aAAa,CAAC;AACxD,SAAK,QAAQ,OAAO;AAAA,EACtB;AAAA;AAAA,EAGA,MAAM,MAAM,OAAsC;AAChD,QAAI,MAAM,WAAW,EAAG,QAAO,CAAC;AAChC,UAAM,MAAM,MAAM,KAAK,OAAO,WAAW,OAAO;AAAA,MAC9C,OAAO,KAAK;AAAA,MACZ,OAAO;AAAA,IACT,CAAC;AAED,WAAO,IAAI,KACR,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,IAAI,CAAC,MAAM,EAAE,SAAqB;AAAA,EACvC;AAAA,EAEA,MAAM,SAAS,MAAiC;AAC9C,UAAM,CAAC,GAAG,IAAI,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC;AACrC,WAAO;AAAA,EACT;AACF;AAMO,SAAS,UAAU,MAAc,OAAqB,CAAC,GAAa;AACzE,QAAM,WAAW,KAAK,YAAY;AAClC,QAAM,UAAU,KAAK,WAAW;AAChC,QAAM,QAAQ,KAAK,QAAQ,SAAS,IAAI,EAAE,KAAK;AAC/C,MAAI,MAAM,UAAU,SAAU,QAAO,QAAQ,CAAC,KAAK,IAAI,CAAC;AAGxD,QAAM,QAAQ,MAAM,MAAM,QAAQ,EAAE,QAAQ,CAAC,MAAM,cAAc,GAAG,QAAQ,CAAC;AAE7E,QAAM,SAAmB,CAAC;AAC1B,MAAI,UAAU;AACd,aAAW,QAAQ,OAAO;AACxB,QAAI,WAAW,QAAQ,SAAS,KAAK,SAAS,IAAI,UAAU;AAC1D,aAAO,KAAK,QAAQ,KAAK,CAAC;AAE1B,gBAAU,UAAU,IAAI,QAAQ,MAAM,CAAC,OAAO,IAAI,SAAS,OAAO;AAAA,IACpE,OAAO;AACL,gBAAU,UAAU,UAAU,SAAS,OAAO;AAAA,IAChD;AAAA,EACF;AACA,MAAI,QAAQ,KAAK,EAAG,QAAO,KAAK,QAAQ,KAAK,CAAC;AAC9C,SAAO;AACT;AAGA,SAAS,cAAc,MAAc,UAA4B;AAC/D,MAAI,KAAK,UAAU,SAAU,QAAO,CAAC,IAAI;AACzC,QAAM,YAAY,KAAK,MAAM,qBAAqB,KAAK,CAAC,IAAI;AAC5D,QAAM,MAAgB,CAAC;AACvB,MAAI,MAAM;AACV,aAAW,KAAK,WAAW;AACzB,QAAI,EAAE,SAAS,UAAU;AACvB,UAAI,KAAK;AACP,YAAI,KAAK,GAAG;AACZ,cAAM;AAAA,MACR;AACA,eAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK,SAAU,KAAI,KAAK,EAAE,MAAM,GAAG,IAAI,QAAQ,CAAC;AAAA,IAChF,WAAW,IAAI,SAAS,EAAE,SAAS,UAAU;AAC3C,UAAI,KAAK,GAAG;AACZ,YAAM;AAAA,IACR,OAAO;AACL,aAAO;AAAA,IACT;AAAA,EACF;AACA,MAAI,IAAK,KAAI,KAAK,GAAG;AACrB,SAAO;AACT;;;AC5EO,IAAM,aAAa;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;AHO1B,IAAM,EAAE,KAAK,IAAI;AAEV,IAAM,cAAN,MAAkB;AAAA,EACf;AAAA,EACA;AAAA,EACC;AAAA,EAET,YAAY,SAA0B,CAAC,GAAG;AACxC,SAAK,SAAS,cAAc,MAAM;AAClC,SAAK,OAAO,IAAI,KAAK,EAAE,kBAAkB,KAAK,OAAO,iBAAiB,CAAC;AACvE,SAAK,WAAW,IAAI,SAAS,KAAK,MAAM;AAAA,EAC1C;AAAA;AAAA,EAGA,MAAM,OAAsB;AAC1B,UAAM,KAAK,KAAK,MAAM,UAAU;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,IAAI,OAAoC;AAC5C,UAAM,WAAW,MAAM,YAAY,KAAK,OAAO,YAAY;AAC3D,UAAM,YAAY,MAAM,aAAa,KAAK,OAAO,aAAa;AAC9D,UAAM,QAAQ,MAAM,SAAS,aAAa,UAAU,SAAS;AAC7D,UAAM,SAAS,MAAM,UAAU;AAC/B,UAAM,WAAW,MAAM,YAAY,CAAC;AAEpC,UAAM,SAAS,UAAU,MAAM,SAAS,MAAM,QAAQ;AACtD,QAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AAEjC,UAAM,UAAU,MAAM,KAAK,SAAS,MAAM,MAAM;AAChD,UAAM,MAAgB,CAAC;AAEvB,UAAM,SAAS,MAAM,KAAK,KAAK,QAAQ;AACvC,QAAI;AACF,YAAM,OAAO,MAAM,OAAO;AAC1B,eAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,cAAM,YACJ,OAAO,SAAS,IACZ,EAAE,GAAG,UAAU,OAAO,GAAG,QAAQ,OAAO,OAAO,IAC/C;AACN,cAAM,EAAE,KAAK,IAAI,MAAM,OAAO;AAAA,UAC5B;AAAA;AAAA,UAEA,CAAC,OAAO,UAAU,WAAW,QAAQ,OAAO,CAAC,GAAG,SAAS,QAAQ,CAAC,CAAC,GAAG,SAAS;AAAA,QACjF;AACA,YAAI,KAAK,KAAK,CAAC,EAAE,EAAE;AAAA,MACrB;AACA,YAAM,OAAO,MAAM,QAAQ;AAAA,IAC7B,SAAS,KAAK;AACZ,YAAM,OAAO,MAAM,UAAU;AAC7B,YAAM;AAAA,IACR,UAAE;AACA,aAAO,QAAQ;AAAA,IACjB;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,aAAa,OAAyD;AAC1E,UAAM,WAAW,MAAM,YAAY,KAAK,OAAO,YAAY;AAC3D,UAAM,YAAY,MAAM,aAAa,KAAK,OAAO,aAAa;AAC9D,UAAM,QAAQ,MAAM,SAAS,aAAa,UAAU,SAAS;AAC7D,UAAM,KAAK,OAAO,EAAE,OAAO,UAAU,YAAY,QAAW,WAAW,aAAa,QAAW,QAAQ,MAAM,OAAO,CAAC;AACrH,WAAO,KAAK,IAAI,KAAK;AAAA,EACvB;AAAA;AAAA,EAGA,MAAM,OAAO,OAAe,OAAsB,CAAC,GAA4B;AAC7E,UAAM,WAAW,KAAK,YAAY,KAAK,OAAO;AAC9C,UAAM,YAAY,KAAK,aAAa,KAAK,OAAO;AAChD,UAAM,gBAAgB,KAAK,0BAA0B;AACrD,UAAM,gBAAgB,KAAK,iBAAiB;AAC5C,UAAM,QAAQ,KAAK,SAAS;AAC5B,UAAM,WAAW,KAAK,YAAY;AAElC,UAAM,WAAW,SAAS,MAAM,KAAK,SAAS,SAAS,KAAK,CAAC;AAI7D,UAAM,YAAsB,CAAC;AAC7B,UAAM,SAAoB,CAAC,QAAQ;AACnC,UAAM,IAAI,CAAC,MAAe,IAAI,OAAO,KAAK,CAAC,CAAC;AAE5C,QAAI,aAAa,OAAO,KAAK,QAAQ,SAAS,GAAG;AAC/C,gBAAU,KAAK,uCAAuC,EAAE,SAAS,CAAC,GAAG;AAAA,IACvE;AACA,QAAI,YAAY,iBAAiB,OAAO,KAAK,QAAQ,QAAQ,GAAG;AAC9D,gBAAU,KAAK,qCAAqC,EAAE,QAAQ,CAAC,GAAG;AAAA,IACpE;AACA,QAAI,iBAAiB,OAAO,KAAK,QAAQ,QAAQ,GAAG;AAClD,gBAAU,KAAK,kBAAkB;AAAA,IACnC;AAEA,UAAM,cAAc,UAAU,SAAS,IAAI,UAAU,KAAK,MAAM,CAAC,MAAM;AAEvE,UAAM,QAAkB,CAAC,WAAW;AACpC,QAAI,KAAK,UAAU;AACjB,YAAM,KAAK,eAAe,EAAE,KAAK,UAAU,KAAK,QAAQ,CAAC,CAAC,SAAS;AAAA,IACrE;AAEA,UAAM,EAAE,KAAK,IAAI,MAAM,KAAK,KAAK;AAAA,MAC/B;AAAA;AAAA;AAAA,eAGS,MAAM,KAAK,OAAO,CAAC;AAAA;AAAA,eAEnB,EAAE,KAAK,CAAC;AAAA,MACjB;AAAA,IACF;AAEA,WAAO,KACJ,IAAI,WAAW,EACf,OAAO,CAAC,MAAM,EAAE,SAAS,QAAQ;AAAA,EACtC;AAAA;AAAA,EAGA,MAAM,OAAO,QAAuC;AAClD,UAAM,QAAkB,CAAC;AACzB,UAAM,SAAoB,CAAC;AAC3B,UAAM,IAAI,CAAC,MAAe,IAAI,OAAO,KAAK,CAAC,CAAC;AAC5C,QAAI,OAAO,GAAI,OAAM,KAAK,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE;AAChD,QAAI,OAAO,MAAO,OAAM,KAAK,WAAW,EAAE,OAAO,KAAK,CAAC,EAAE;AACzD,QAAI,OAAO,SAAU,OAAM,KAAK,eAAe,EAAE,OAAO,QAAQ,CAAC,EAAE;AACnE,QAAI,OAAO,UAAW,OAAM,KAAK,gBAAgB,EAAE,OAAO,SAAS,CAAC,EAAE;AACtE,QAAI,OAAO,OAAQ,OAAM,KAAK,YAAY,EAAE,OAAO,MAAM,CAAC,EAAE;AAC5D,QAAI,MAAM,WAAW,GAAG;AACtB,YAAM,IAAI,MAAM,oFAAoF;AAAA,IACtG;AACA,UAAM,EAAE,SAAS,IAAI,MAAM,KAAK,KAAK;AAAA,MACnC,+BAA+B,MAAM,KAAK,OAAO,CAAC;AAAA,MAClD;AAAA,IACF;AACA,WAAO,YAAY;AAAA,EACrB;AAAA;AAAA,EAGA,MAAM,QAAuB;AAC3B,UAAM,KAAK,KAAK,IAAI;AAAA,EACtB;AACF;AAEO,SAAS,kBAAkB,QAAuC;AACvE,SAAO,IAAI,YAAY,MAAM;AAC/B;AAEA,SAAS,aAAa,UAAyB,WAAiC;AAC9E,MAAI,UAAW,QAAO;AACtB,MAAI,SAAU,QAAO;AACrB,SAAO;AACT;AAEA,SAAS,OAAO,QAA6B,OAAuB;AAClE,SAAO,CAAC,UAAU,OAAO,SAAS,KAAK;AACzC;AAGA,SAAS,SAAS,KAAuB;AACvC,SAAO,IAAI,IAAI,KAAK,GAAG,CAAC;AAC1B;AAEA,SAAS,YAAY,KAAwB;AAC3C,SAAO,EAAE,GAAG,YAAY,GAAG,GAAG,OAAO,OAAO,IAAI,KAAK,EAAE;AACzD;AAEA,SAAS,YAAY,KAA2B;AAC9C,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,UAAU,IAAI;AAAA,IACd,WAAW,IAAI;AAAA,IACf,QAAQ,IAAI;AAAA,IACZ,SAAS,IAAI;AAAA,IACb,UAAU,IAAI,YAAY,CAAC;AAAA,IAC3B,WAAW,IAAI,sBAAsB,OAAO,IAAI,WAAW,YAAY,IAAI,IAAI;AAAA,EACjF;AACF;","names":[]}

package/migrations/001_init_knowledge.sql

@@ -0,0 +1,30 @@
+-- AI Knowledge DB schema (Postgres + pgvector)
+-- Run once against your Hetzner/EasyPanel Postgres instance, or via kb.init().
+-- The vector dimension below matches text-embedding-3-small (1536).
+-- If you switch to text-embedding-3-large, change 1536 -> 3072 and re-index.
+
+create extension if not exists vector;
+create extension if not exists "pgcrypto"; -- for gen_random_uuid()
+
+create table if not exists knowledge (
+  id          uuid primary key default gen_random_uuid(),
+  scope       text not null check (scope in ('global', 'client', 'project')),
+  client_id   text,
+  project_id  text,
+  source      text,
+  content     text not null,
+  embedding   vector(1536) not null,
+  metadata    jsonb not null default '{}',
+  created_at  timestamptz not null default now()
+);
+
+-- Scope filtering happens on every search, so index the columns we filter by.
+create index if not exists knowledge_client_idx  on knowledge (client_id);
+create index if not exists knowledge_project_idx on knowledge (project_id);
+create index if not exists knowledge_scope_idx   on knowledge (scope);
+create index if not exists knowledge_metadata_idx on knowledge using gin (metadata);
+
+-- Approximate nearest-neighbour index for fast cosine search.
+-- HNSW gives good recall/speed without tuning lists like IVFFlat.
+create index if not exists knowledge_embedding_idx
+  on knowledge using hnsw (embedding vector_cosine_ops);

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@digitalvibes/ai-knowledge-db",
+  "version": "0.1.0",
+  "description": "AI knowledge vector storage (Postgres + pgvector) for Dibe website/project work. Stores client- and project-scoped knowledge and serves it back via semantic search.",
+  "type": "module",
+  "license": "MIT",
+  "packageManager": "pnpm@9.15.2",
+  "publishConfig": {
+    "access": "public"
+  },
+  "sideEffects": false,
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs",
+      "default": "./dist/index.js"
+    },
+    "./schema.sql": "./migrations/001_init_knowledge.sql"
+  },
+  "files": [
+    "dist",
+    "migrations",
+    "skills",
+    "README.md"
+  ],
+  "bin": {
+    "knowledge-db": "./dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "typecheck": "tsc --noEmit",
+    "prepack": "pnpm build",
+    "prepublishOnly": "pnpm typecheck && pnpm build"
+  },
+  "keywords": [
+    "vector",
+    "embeddings",
+    "pgvector",
+    "rag",
+    "knowledge-base",
+    "openai",
+    "ai"
+  ],
+  "author": "Dibe",
+  "dependencies": {
+    "openai": "^4.77.0",
+    "pg": "^8.13.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.0",
+    "@types/pg": "^8.11.10",
+    "tsup": "^8.3.5",
+    "typescript": "^5.7.2"
+  },
+  "engines": {
+    "node": ">=18"
+  }
+}

package/skills/knowledge-db/SKILL.md

@@ -0,0 +1,87 @@
+---
+name: knowledge-db
+description: Store and retrieve client- and project-scoped knowledge from the Dibe AI Knowledge vector DB (Postgres + pgvector). Use whenever you need durable project/client context — brand voice, decisions, specs, past work — or when the user says "remember this", "save to knowledge", "what do we know about X", "look up the client's …", or is building a website/app that should pull from stored knowledge.
+---
+
+# Dibe AI Knowledge DB
+
+A small vector store (`@digitalvibes/ai-knowledge-db`) that holds **client**, **project**, and **global** knowledge and serves it back via semantic search. Use it to persist context that should survive across sessions and to retrieve relevant context before doing work.
+
+## When to use this
+
+- **Retrieve** before building: pull brand voice, design tokens, prior decisions, client facts before writing copy/components.
+- **Add** durable facts: "remember that the client prefers X", project specs, naming conventions, completed-work notes.
+- **Update** a source that changed: re-ingest a file/URL idempotently (no duplicates).
+
+Do NOT use it for throwaway chat context or secrets/credentials.
+
+## Setup (once per consuming project)
+
+The package reads everything from the project's own `.env` — nothing is hardcoded:
+
+```
+KNOWLEDGE_DB_URL=postgres://…        # Hetzner/EasyPanel pgvector instance
+OPENAI_API_KEY=sk-…
+KNOWLEDGE_CLIENT_ID=acme-corp        # default client for this repo
+KNOWLEDGE_PROJECT_ID=acme-website    # default project for this repo
+```
+
+Initialise the schema once: `npx knowledge-db init`
+
+## Scopes (the mental model)
+
+| scope | meaning | example |
+|-------|---------|---------|
+| `global` | applies to every client/project | Dibe's own conventions, reusable patterns |
+| `client` | shared across all of one client's projects | brand voice, logo rules, tone of voice |
+| `project` | specific to one project | this site's sitemap, copy decisions, specs |
+
+A search for a project automatically also pulls that **client's** shared knowledge and **global** knowledge, ranked together by relevance.
+
+## CLI (preferred for quick ops)
+
+```bash
+# retrieve — scoping comes from env defaults unless overridden
+npx knowledge-db search "what is the brand voice?"
+npx knowledge-db search "homepage hero copy" --project acme-website --json
+
+# add a fact (scope auto-derives: project if KNOWLEDGE_PROJECT_ID set, else client, else global)
+npx knowledge-db add "Client prefers sentence case for all headings." --scope client
+npx knowledge-db add "Hero CTA finalised as 'Start free'." --source decisions.md
+
+# add / update a file or doc idempotently (re-running replaces prior rows for that source)
+npx knowledge-db add-file ./brand/voice.md --scope client --source brand/voice.md
+
+# remove
+npx knowledge-db delete --source brand/voice.md
+npx knowledge-db delete --id <uuid>
+```
+
+`--json` on `search` returns structured results (id, scope, source, score, content) — use it when you need to feed results back into reasoning.
+
+## Library (for app/server code)
+
+```ts
+import { createKnowledgeDB } from "@digitalvibes/ai-knowledge-db";
+
+const kb = createKnowledgeDB(); // reads .env
+
+// retrieve relevant context before generating
+const hits = await kb.search("brand voice and tone", { limit: 5 });
+const context = hits.map((h) => h.content).join("\n\n");
+
+// store a durable fact
+await kb.add({ content: "Client approved the green palette.", scope: "client" });
+
+// idempotently re-ingest a doc
+await kb.upsertSource({ content: fileText, source: "specs/sitemap.md" });
+
+await kb.close();
+```
+
+## Workflow guidance for the agent
+
+1. **Before** building website copy/components or answering a "what do we know" question, run a `search` and ground your output in the results. Cite the `source` when relevant.
+2. **After** the user states a durable preference/decision ("remember…", "from now on…", "the client wants…"), `add` it with the right scope. Prefer `client` scope for anything reusable across the client's projects; `project` for this build only.
+3. When re-ingesting a known file/URL, use `add-file` / `upsertSource` (not `add`) so you don't create duplicates.
+4. If a search returns nothing useful, say so rather than inventing context.