@hasna/knowledge 0.2.18 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -9
- package/bin/open-knowledge-mcp.js +292 -15
- package/bin/open-knowledge.js +99 -58
- package/docs/architecture/ai-native-knowledge-base.md +7 -0
- package/docs/architecture/hybrid-semantic-search.md +10 -1
- package/package.json +1 -1
- package/src/cli.ts +46 -12
- package/src/knowledge-db.ts +27 -1
- package/src/mcp.js +44 -0
- package/src/reindex.ts +260 -0
- package/src/service.ts +28 -0
package/bin/open-knowledge.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
|
-
var
|
|
4
|
-
`);return t}function
|
|
3
|
+
var O=import.meta.require;import{readFileSync as _e,writeFileSync as ue,existsSync as le,renameSync as Or,unlinkSync as at}from"fs";import{randomUUID as ct}from"crypto";import{existsSync as br,mkdirSync as Ce,readFileSync as vr,writeFileSync as wr}from"fs";import{homedir as nt}from"os";import{dirname as Tr,join as C,resolve as xr}from"path";var ne=C(".hasna","apps","knowledge");function Pe(){return C(nt(),".open-knowledge","db.json")}function De(){return C(nt(),".hasna","apps","knowledge")}function Sr(e=process.cwd()){return xr(e,ne)}function re(e){return{home:e,configPath:C(e,"config.json"),jsonStorePath:C(e,"db.json"),knowledgeDbPath:C(e,"knowledge.db"),artifactsDir:C(e,"artifacts"),cacheDir:C(e,"cache"),exportsDir:C(e,"exports"),indexesDir:C(e,"indexes"),logsDir:C(e,"logs"),runsDir:C(e,"runs"),schemasDir:C(e,"schemas"),wikiDir:C(e,"wiki")}}function Rr(){return{version:1,mode:"local",storage:{type:"local",artifacts_root:"artifacts"},sources:{preferred_ref:"open-files",allowed_schemes:["open-files","s3","file","https","http"]},providers:{default_model:"openai:gpt-5.2",aliases:{fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"},openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},embeddings:{default_model:"openai:text-embedding-3-small",dimensions:1536,batch_size:64,max_parallel_calls:4},safety:{network:{web_search_enabled:!1,s3_reads_enabled:!1,allowed_s3_buckets:[]},redaction:{enabled:!0},approvals:{generated_writes_require_approval:!0}}}}function it(e){let t=re(e);Ce(t.home,{recursive:!0});for(let r of[t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir])Ce(r,{recursive:!0});if(!br(t.configPath))wr(t.configPath,`${JSON.stringify(Rr(),null,2)}
|
|
4
|
+
`);return t}function st(e,t=process.cwd()){if(e==="project"||e==="local")return re(Sr(t));return re(De())}function de(e){Ce(Tr(e),{recursive:!0})}function ot(e){let t=vr(e,"utf8");return JSON.parse(t)}function Ue(){return re(De()).jsonStorePath}function Me(e){if(!le(e))if(de(e),e===Ue()&&le(Pe()))ue(e,_e(Pe(),"utf8"));else ue(e,JSON.stringify({items:[]},null,2))}function Nr(e){return`${e}.lock`}function Ar(e,t){let i=Date.now();while(Date.now()-i<5000){try{if(!le(e)){ue(e,JSON.stringify({owner:t,ts:Date.now()}));return}let c=JSON.parse(_e(e,"utf8"));if(Date.now()-c.ts>1e4)at(e)}catch{}let s=Date.now();while(Date.now()-s<50);}throw Error(`Could not acquire lock on ${e} after 5000ms`)}function Ir(e,t){try{if(le(e)){if(JSON.parse(_e(e,"utf8")).owner===t)at(e)}}catch{}}function P(e){Me(e);let t=_e(e,"utf8"),r=JSON.parse(t);if(!r||!Array.isArray(r.items))return{items:[]};return r}function K(e,t){let r=`${e}.tmp.${ct()}`;ue(r,JSON.stringify(t,null,2)),Or(r,e)}function D(e,t){let r=ct(),n=Nr(e);Ar(n,r);try{return t()}finally{Ir(n,r)}}function je(){return`k_${Date.now().toString(36)}_${Math.random().toString(36).slice(2,8)}`}function dt(e){return e.replace(/^k_/,"").slice(0,12)}import{Database as Lr}from"bun:sqlite";var Cr=`
|
|
5
5
|
PRAGMA journal_mode = WAL;
|
|
6
6
|
PRAGMA foreign_keys = ON;
|
|
7
7
|
|
|
@@ -168,7 +168,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
168
168
|
|
|
169
169
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
170
170
|
VALUES (1, datetime('now'));
|
|
171
|
-
`,
|
|
171
|
+
`,Pr=`
|
|
172
172
|
DROP TABLE IF EXISTS chunks_fts;
|
|
173
173
|
|
|
174
174
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
@@ -181,7 +181,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
181
181
|
|
|
182
182
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
183
183
|
VALUES (2, datetime('now'));
|
|
184
|
-
`,
|
|
184
|
+
`,Dr=`
|
|
185
185
|
CREATE TABLE IF NOT EXISTS audit_events (
|
|
186
186
|
id TEXT PRIMARY KEY,
|
|
187
187
|
event_type TEXT NOT NULL,
|
|
@@ -212,7 +212,7 @@ CREATE INDEX IF NOT EXISTS idx_approval_gates_status ON approval_gates(status);
|
|
|
212
212
|
|
|
213
213
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
214
214
|
VALUES (3, datetime('now'));
|
|
215
|
-
`,
|
|
215
|
+
`,Ur=`
|
|
216
216
|
CREATE TABLE IF NOT EXISTS vector_index_entries (
|
|
217
217
|
id TEXT PRIMARY KEY,
|
|
218
218
|
chunk_id TEXT NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
|
|
@@ -243,8 +243,29 @@ CREATE INDEX IF NOT EXISTS idx_vector_index_status ON vector_index_entries(statu
|
|
|
243
243
|
|
|
244
244
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
245
245
|
VALUES (4, datetime('now'));
|
|
246
|
-
|
|
247
|
-
|
|
246
|
+
`,Mr=`
|
|
247
|
+
CREATE TABLE IF NOT EXISTS reindex_queue (
|
|
248
|
+
id TEXT PRIMARY KEY,
|
|
249
|
+
kind TEXT NOT NULL,
|
|
250
|
+
target_id TEXT NOT NULL,
|
|
251
|
+
source_uri TEXT,
|
|
252
|
+
reason TEXT NOT NULL,
|
|
253
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
254
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
255
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
256
|
+
created_at TEXT NOT NULL,
|
|
257
|
+
updated_at TEXT NOT NULL,
|
|
258
|
+
UNIQUE(kind, target_id, reason)
|
|
259
|
+
);
|
|
260
|
+
|
|
261
|
+
CREATE INDEX IF NOT EXISTS idx_reindex_queue_status ON reindex_queue(status);
|
|
262
|
+
CREATE INDEX IF NOT EXISTS idx_reindex_queue_kind_target ON reindex_queue(kind, target_id);
|
|
263
|
+
CREATE INDEX IF NOT EXISTS idx_reindex_queue_source_uri ON reindex_queue(source_uri);
|
|
264
|
+
|
|
265
|
+
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
266
|
+
VALUES (5, datetime('now'));
|
|
267
|
+
`;function b(e){de(e);let t=new Lr(e);return t.exec("PRAGMA foreign_keys = ON;"),t.exec("PRAGMA busy_timeout = 5000;"),t}function x(e){let t=b(e);try{if(t.exec(Cr),V(t)<2)t.exec(Pr);if(V(t)<3)t.exec(Dr);if(V(t)<4)t.exec(Ur);if(V(t)<5)t.exec(Mr);return{path:e,schema_version:V(t)}}finally{t.close()}}function V(e){return e.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0}function I(e,t){return e.query(`SELECT COUNT(*) AS n FROM ${t}`).get()?.n??0}function ut(e){let t=b(e);try{return{schema_version:V(t),sources:I(t,"sources"),source_revisions:I(t,"source_revisions"),chunks:I(t,"chunks"),wiki_pages:I(t,"wiki_pages"),citations:I(t,"citations"),indexes:I(t,"knowledge_indexes"),runs:I(t,"runs"),run_events:I(t,"run_events"),redaction_findings:I(t,"redaction_findings"),audit_events:I(t,"audit_events"),approval_gates:I(t,"approval_gates"),storage_objects:I(t,"storage_objects"),embeddings:I(t,"chunk_embeddings"),vector_entries:I(t,"vector_index_entries"),reindex_queue:I(t,"reindex_queue")}}finally{t.close()}}import{existsSync as jr,mkdirSync as lt,readFileSync as Kr,writeFileSync as Fr}from"fs";import{dirname as Wr,join as Ke,relative as Xr,sep as $r}from"path";function ie(e){let t=e.replace(/\\/g,"/").trim();if(!t||t.startsWith("/"))throw Error(`Invalid artifact key: ${e}`);let r=t.split("/").filter(Boolean);if(r.length===0||r.some((n)=>n==="."||n===".."))throw Error(`Invalid artifact key: ${e}`);return r.join("/")}function Fe(e,t){let r=Xr(e,t);if(r.startsWith("..")||r===".."||r.startsWith(`..${$r}`))throw Error(`Artifact path escapes root: ${t}`)}class _t{root;type="local";canRead=!0;canWrite=!0;constructor(e){this.root=e;lt(e,{recursive:!0})}async put(e){let t=ie(e.key),r=Ke(this.root,t);return Fe(this.root,r),lt(Wr(r),{recursive:!0}),Fr(r,e.body),{key:t,uri:`file://${r}`}}async getText(e){let t=ie(e),r=Ke(this.root,t);return Fe(this.root,r),Kr(r,"utf8")}async exists(e){let t=ie(e),r=Ke(this.root,t);return Fe(this.root,r),jr(r)}}class ft{options;type="s3";canRead=!0;canWrite=!0;client;constructor(e){this.options=e;this.client=e.client}async getClient(){if(this.client)return this.client;let[{S3Client:e},{fromIni:t}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]);return this.client=new e({region:this.options.region,credentials:this.options.profile?t({profile:this.options.profile}):void 0,maxAttempts:this.options.max_attempts}),this.client}objectKey(e){let t=ie(e),r=this.options.prefix?ie(this.options.prefix):"";return r?`${r}/${t}`:t}async put(e){let[{PutObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e.key);return await r.send(new t({Bucket:this.options.bucket,Key:n,Body:e.body,ContentType:e.content_type,Metadata:e.metadata,ServerSideEncryption:this.options.server_side_encryption,SSEKMSKeyId:this.options.kms_key_id})),{key:n,uri:`s3://${this.options.bucket}/${n}`}}async getText(e){let[{GetObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e),i=await r.send(new t({Bucket:this.options.bucket,Key:n}));if(!i.Body)return"";return await i.Body.transformToString()}async exists(e){let[{HeadObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e);try{return await r.send(new t({Bucket:this.options.bucket,Key:n})),!0}catch(i){let s=i instanceof Error?i.name:"";if(s==="NotFound"||s==="NoSuchKey"||s==="NotFoundError")return!1;throw i}}}function gt(e,t){if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)throw Error("S3 artifact storage requires storage.s3.bucket");return new ft({bucket:e.storage.s3.bucket,prefix:e.storage.s3.prefix,region:e.storage.s3.region,profile:e.storage.s3.profile,max_attempts:e.storage.s3.max_attempts,server_side_encryption:e.storage.s3.server_side_encryption,kms_key_id:e.storage.s3.kms_key_id})}return new _t(t.artifactsDir)}import{randomUUID as jt}from"crypto";import{randomUUID as qr}from"crypto";var We={openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},Hr={openai:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!0,reasoning:!0,embeddings:!0},anthropic:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!1,reasoning:!0,embeddings:!1},deepseek:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!1,native_web_search:!1,reasoning:!0,embeddings:!1}},Br={default:"openai:gpt-5.2",fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"};function mt(e){return e?.providers??{}}function W(e,t){let r=mt(e)[t]??{};return{...We[t],...r}}function ht(e){let t=mt(e);return{...Br,...t.default_model?{default:t.default_model}:{},...t.aliases??{}}}function L(e){let[t,...r]=e.split(":"),n=r.join(":");if(t!=="openai"&&t!=="anthropic"&&t!=="deepseek")throw Error(`Unsupported AI provider: ${t}`);if(!n)throw Error(`Invalid model ref: ${e}. Expected provider:model.`);return{provider:t,model:n}}function X(e,t){return ht(t)[e]??e}function Xe(e){let t=ht(e);return Object.entries(t).map(([r,n])=>{let i=L(n);return{alias:r,model_ref:n,provider:i.provider,model:i.model,default:r==="default",capabilities:Hr[i.provider]}})}function Et(e,t=process.env){return Object.keys(We).map((r)=>{let n=W(e,r),i=Boolean(t[n.api_key_env]);return{provider:r,api_key_env:n.api_key_env,configured:i,source:i?"env":"missing",base_url:n.base_url??null,default_model:n.default_model}})}function yt(e,t=process.env){return{default_model:X("default",e),providers:Et(e,t),models:Xe(e)}}function B(e,t,r=process.env){let n=Et(t,r).find((i)=>i.provider===e);if(!n)throw Error(`Unsupported AI provider: ${e}`);if(!n.configured)throw Error(`Missing ${n.api_key_env} for ${e}. Set the env var to use this provider.`);return n}async function zr(e){if(e==="openai"){let{createOpenAI:r}=await import("@ai-sdk/openai");return r}if(e==="anthropic"){let{createAnthropic:r}=await import("@ai-sdk/anthropic");return r}let{createDeepSeek:t}=await import("@ai-sdk/deepseek");return t}async function Jr(e={}){let{createProviderRegistry:t}=await import("ai"),r=e.env??process.env,n={};for(let i of Object.keys(We)){let s=W(e.config,i),c=r[s.api_key_env];if(!c)continue;let u=e.factories?.[i]??await zr(i);n[i]=u({apiKey:c,baseURL:s.base_url})}return t(n)}async function kt(e,t={}){let r=X(e,t.config),n=L(r);return B(n.provider,t.config,t.env),(await Jr(t)).languageModel(r)}function pt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return 0}function fe(e){let t=e.usage??{};return{provider:e.provider,model:e.model,input_tokens:pt(t,["inputTokens","promptTokens","input_tokens","prompt_tokens"]),output_tokens:pt(t,["outputTokens","completionTokens","output_tokens","completion_tokens"]),cost_usd:e.costUsd??0,metadata:{usage:t,provider_metadata:e.providerMetadata??{}}}}function ge(e,t){let r=`usage_${qr()}`;return e.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
268
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[r,t.run_id??null,t.provider,t.model,t.input_tokens,t.output_tokens,t.cost_usd,JSON.stringify(t.metadata),t.created_at??new Date().toISOString()]),r}import{createHash as mn}from"crypto";function $e(e){return["deleted","stale","invalidated","reindex_required"].includes((e??"").toLowerCase())}function $(e){let t=e.status??null;return{source_owner:"open-files",source_ref:e.source_ref??null,source_uri:e.source_uri??null,source_kind:e.source_kind??null,source_revision_id:e.source_revision_id??null,revision:e.revision??null,hash:e.hash??null,chunk_id:e.chunk_id??null,start_offset:e.start_offset??null,end_offset:e.end_offset??null,status:t,read_only:!0,citation_required:!0,resolver:e.resolver??null,stale:$e(t)}}function qe(e){return{source_owner:"open-files",generated_from:e.generated_from,artifact_key:e.artifact_key,source_refs:e.source_refs??[],read_only_sources:!0,citation_required:e.citation_required??!0,raw_source_bytes_stored_in_open_knowledge:!1}}function bt(e,t){return{...e,provenance:t}}import{createHash as Tt}from"crypto";var Gr="openai:text-embedding-3-small",xt=1536;function pe(e){return e?.embeddings??{}}function vt(e,t){return`${e}_${Tt("sha256").update(t).digest("hex").slice(0,20)}`}function Be(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function j(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function wt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function He(e){return Math.sqrt(e.reduce((t,r)=>t+r*r,0))}function Yr(e,t,r=He(t)){let n=He(e);if(n===0||r===0)return 0;let i=Math.min(e.length,t.length),s=0;for(let c=0;c<i;c+=1)s+=e[c]*t[c];return s/(n*r)}function Vr(e,t){let r=Tt("sha256").update(e).digest();return Array.from({length:t},(n,i)=>{let s=r[i%r.length]/255;return Number((s*2-1).toFixed(6))})}async function Qr(e,t,r=process.env){B("openai",t,r);let n=W(t,"openai"),{createOpenAI:i}=await import("@ai-sdk/openai"),s=i({apiKey:r[n.api_key_env],baseURL:n.base_url});if(s.embeddingModel)return s.embeddingModel(e);if(s.textEmbedding)return s.textEmbedding(e);if(s.textEmbeddingModel)return s.textEmbeddingModel(e);throw Error("OpenAI provider does not expose an embedding model factory.")}function z(e,t){if(!e||e==="default"||e==="embedding")return pe(t).default_model??Gr;return e}async function St(e,t={}){let r=z(t.modelRef,t.config),n=L(r);if(n.provider!=="openai")throw Error(`Embedding provider ${n.provider} is not supported yet. Use openai:text-embedding-3-small.`);let i=t.dimensions??pe(t.config).dimensions??xt;if(t.fake)return{provider:n.provider,model:n.model,dimensions:i,vectors:e.map((a)=>Vr(a,i)),usage:{input_tokens:e.reduce((a,d)=>a+Math.max(1,Math.ceil(d.split(/\s+/).filter(Boolean).length*1.25)),0)}};let{embedMany:s}=await import("ai"),c=await Qr(n.model,t.config,t.env),u=await s({model:c,values:e,maxParallelCalls:t.maxParallelCalls??pe(t.config).max_parallel_calls,providerOptions:{openai:{dimensions:i}}}),_=u.embeddings;return{provider:n.provider,model:n.model,dimensions:_[0]?.length??i,vectors:_,usage:{input_tokens:u.usage?.tokens??0}}}function Zr(e,t){if(t.sourceRevisionId)return e.query(`SELECT
|
|
248
269
|
c.id,
|
|
249
270
|
c.text,
|
|
250
271
|
c.token_count,
|
|
@@ -282,7 +303,7 @@ VALUES (4, datetime('now'));
|
|
|
282
303
|
ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
|
|
283
304
|
WHERE v.id IS NULL
|
|
284
305
|
ORDER BY c.created_at ASC, c.ordinal ASC
|
|
285
|
-
LIMIT ?`).all(t.provider,t.model,t.limit)}function
|
|
306
|
+
LIMIT ?`).all(t.provider,t.model,t.limit)}function en(e){let t=Be(e.metadata_json),r=t.provenance;if(r&&typeof r==="object"&&!Array.isArray(r))return r;return $({source_ref:j(t,["source_ref"]),source_uri:e.source_uri??j(t,["source_uri"]),source_kind:e.source_kind??j(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??j(t,["revision"]),hash:e.hash??j(t,["hash"]),chunk_id:e.id,start_offset:e.start_offset??wt(t,["start_offset"]),end_offset:e.end_offset??wt(t,["end_offset"]),status:j(t,["status"]),resolver:"open-files-read-only"})}function tn(e,t,r,n){let i=e.prepare(`
|
|
286
307
|
INSERT INTO chunk_embeddings (id, chunk_id, provider, model, dimensions, vector_json, created_at)
|
|
287
308
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
288
309
|
ON CONFLICT(chunk_id, provider, model) DO UPDATE SET
|
|
@@ -311,10 +332,10 @@ VALUES (4, datetime('now'));
|
|
|
311
332
|
status = excluded.status,
|
|
312
333
|
metadata_json = excluded.metadata_json,
|
|
313
334
|
updated_at = excluded.updated_at
|
|
314
|
-
`);return e.transaction(()=>{for(let u=0;u<t.length;u+=1){let _=t[u],a=r.vectors[u];if(!a)continue;let d=
|
|
335
|
+
`);return e.transaction(()=>{for(let u=0;u<t.length;u+=1){let _=t[u],a=r.vectors[u];if(!a)continue;let d=Be(_.metadata_json),o=en(_),l=o.source_ref??j(d,["source_ref"]),f=o.source_uri??_.source_uri??j(d,["source_uri"]),g=o.revision??_.revision??j(d,["revision"]),E=o.hash??_.hash??j(d,["hash"]),h=o.status??j(d,["status"])??"active",m=JSON.stringify(a);i.run(vt("emb",`${_.id}\x00${r.provider}\x00${r.model}`),_.id,r.provider,r.model,r.dimensions,m,n),s.run(vt("vec",`${_.id}\x00${r.provider}\x00${r.model}`),_.id,_.source_revision_id,r.provider,r.model,r.dimensions,m,He(a),f,l,g,E,o.start_offset,o.end_offset,_.token_count,h,JSON.stringify({...d,provenance:o,embedded_at:n}),n,n)}})(),t.length}async function me(e){let t=z(e.modelRef,e.config),r=L(t);if(r.provider!=="openai")throw Error(`Embedding provider ${r.provider} is not supported yet.`);let n=(e.now??new Date).toISOString(),i=Math.max(1,Math.min(e.limit??100,1000));x(e.dbPath);let s=b(e.dbPath),c;try{c=Zr(s,{provider:r.provider,model:r.model,limit:i,sourceRevisionId:e.sourceRevisionId})}finally{s.close()}if(c.length===0)return{provider:r.provider,model:r.model,dimensions:e.dimensions??pe(e.config).dimensions??xt,chunks_seen:0,chunks_embedded:0,embeddings_upserted:0,vector_entries_upserted:0,usage:{input_tokens:0}};let u=await St(c.map((a)=>a.text),e),_=b(e.dbPath);try{let a=tn(_,c,u,n);return{provider:u.provider,model:u.model,dimensions:u.dimensions,chunks_seen:c.length,chunks_embedded:c.length,embeddings_upserted:a,vector_entries_upserted:a,usage:u.usage}}finally{_.close()}}function Rt(e){x(e);let t=b(e);try{let r=t.query("SELECT COUNT(*) AS n FROM chunk_embeddings").get()?.n??0,n=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0,i=t.query(`SELECT provider, model, dimensions, COUNT(*) AS entries, MAX(updated_at) AS updated_at
|
|
315
336
|
FROM vector_index_entries
|
|
316
337
|
GROUP BY provider, model, dimensions
|
|
317
|
-
ORDER BY provider, model`).all();return{total_embeddings:r,total_vector_entries:n,indexes:i}}finally{t.close()}}async function
|
|
338
|
+
ORDER BY provider, model`).all();return{total_embeddings:r,total_vector_entries:n,indexes:i}}finally{t.close()}}async function he(e){let t=z(e.modelRef,e.config),r=L(t),n=Math.max(1,Math.min(e.limit??10,100)),i=await St([e.query],e),s=i.vectors[0]??[];x(e.dbPath);let c=b(e.dbPath);try{let _=c.query(`SELECT
|
|
318
339
|
v.chunk_id,
|
|
319
340
|
c.text,
|
|
320
341
|
v.vector_json,
|
|
@@ -326,7 +347,7 @@ VALUES (4, datetime('now'));
|
|
|
326
347
|
v.metadata_json
|
|
327
348
|
FROM vector_index_entries v
|
|
328
349
|
JOIN chunks c ON c.id = v.chunk_id
|
|
329
|
-
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(r.provider,r.model).map((a)=>{let d=JSON.parse(a.vector_json),o=
|
|
350
|
+
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(r.provider,r.model).map((a)=>{let d=JSON.parse(a.vector_json),o=Be(a.metadata_json),l=o.provenance&&typeof o.provenance==="object"&&!Array.isArray(o.provenance)?o.provenance:null;return{chunk_id:a.chunk_id,score:Yr(s,d,a.vector_norm),text:a.text,source_uri:a.source_uri,source_ref:a.source_ref,revision:a.revision,hash:a.hash,provenance:l}}).sort((a,d)=>d.score-a.score).slice(0,n);return{provider:r.provider,model:r.model,dimensions:i.dimensions,query:e.query,results:_}}finally{c.close()}}function ye(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function U(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function Ot(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Nt(e){return Array.from(new Set(e))}function rn(e){let t=e.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}_]+/gu)??[];return Nt(t.filter((r)=>r.length>0)).slice(0,16)}function nn(e){if(e.length===0)return null;return e.map((t)=>`${t}*`).join(" OR ")}function sn(e){return e.replace(/[\\%_]/g,(t)=>`\\${t}`)}function At(e,t){return e.flatMap((r)=>Array.from({length:t},()=>`%${sn(r)}%`))}function on(e,t){let r=Number.isFinite(e)?1/(1+Math.abs(e)):0,n=1/(1+t);return ke(Math.max(r,n))}function It(e,t){if(t.length===0)return 0;let r=t.filter((n)=>e.includes(n)).length;if(r===0)return 0;return ke(Math.min(0.85,0.35+r/t.length*0.5))}function an(e){return ke(Math.max(0,Math.min(1,(e+1)/2)))}function ke(e){return Number(e.toFixed(6))}function se(e,t){let r=e.keyword??0,n=e.semantic??0,i=e.catalog??0,s=t?.chunk_id?0.05:0;return ke(Math.min(1,r*0.55+n*0.4+i*0.35+s))}function ze(e){let t=e.provenance;return t&&typeof t==="object"&&!Array.isArray(t)?t:null}function cn(e){let t=ye(e.chunk_metadata_json),r=ze(t);if(r)return r;if(!e.source_revision_id&&!e.source_uri)return null;return $({source_ref:U(t,["source_ref"]),source_uri:e.source_uri??U(t,["source_uri"]),source_kind:e.source_kind??U(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??U(t,["revision"]),hash:e.hash??U(t,["hash"]),chunk_id:e.chunk_id,start_offset:e.start_offset??Ot(t,["start_offset"]),end_offset:e.end_offset??Ot(t,["end_offset"]),status:U(t,["status"]),resolver:"open-files-read-only"})}function dn(e,t,r){if(!t)return[];return e.query(`SELECT
|
|
330
351
|
chunks_fts.chunk_id,
|
|
331
352
|
c.kind AS chunk_kind,
|
|
332
353
|
c.wiki_page_id,
|
|
@@ -355,91 +376,110 @@ VALUES (4, datetime('now'));
|
|
|
355
376
|
LEFT JOIN wiki_pages wp ON wp.id = c.wiki_page_id
|
|
356
377
|
WHERE chunks_fts MATCH ?
|
|
357
378
|
ORDER BY rank ASC
|
|
358
|
-
LIMIT ?`).all(t,r)}function
|
|
379
|
+
LIMIT ?`).all(t,r)}function Lt(e,t){if(t.length===0)return"1 = 0";return t.map(()=>`(${e.map((n)=>`lower(COALESCE(${n}, '')) LIKE ? ESCAPE '\\'`).join(" OR ")})`).join(" OR ")}function un(e,t,r){let n=["path","title","artifact_uri","metadata_json"];return e.query(`SELECT id, path, title, artifact_uri, content_hash, status, metadata_json
|
|
359
380
|
FROM wiki_pages
|
|
360
|
-
WHERE status = 'active' AND (${
|
|
381
|
+
WHERE status = 'active' AND (${Lt(n,t)})
|
|
361
382
|
ORDER BY updated_at DESC
|
|
362
|
-
LIMIT ?`).all(...
|
|
383
|
+
LIMIT ?`).all(...At(t,n.length),r)}function ln(e,t,r){let n=["kind","name","shard_key","artifact_uri","metadata_json"];return e.query(`SELECT id, kind, name, artifact_uri, shard_key, metadata_json
|
|
363
384
|
FROM knowledge_indexes
|
|
364
|
-
WHERE ${
|
|
385
|
+
WHERE ${Lt(n,t)}
|
|
365
386
|
ORDER BY updated_at DESC
|
|
366
|
-
LIMIT ?`).all(...
|
|
387
|
+
LIMIT ?`).all(...At(t,n.length),r)}function _n(e,t){let r=ye(e.chunk_metadata_json),n=cn(e),i=U(r,["source_ref"]),s=e.source_uri??U(r,["source_uri"]),c=Boolean(e.wiki_page_id),u={kind:c?"wiki_chunk":"source_chunk",id:e.chunk_id,title:c?e.wiki_title:e.source_title,text:e.text,score:0,scores:{keyword:t},source:s||i?{uri:s,ref:i,kind:e.source_kind??U(r,["source_kind"]),revision:e.revision??U(r,["revision"]),hash:e.hash??U(r,["hash"])}:null,citation:{chunk_id:e.chunk_id,start_offset:e.start_offset,end_offset:e.end_offset},artifact:c?{uri:e.wiki_artifact_uri,path:e.wiki_path,hash:e.wiki_content_hash,shard_key:e.wiki_path}:null,provenance:n,reasons:["keyword_match"]};return u.score=se(u.scores,u.citation),u}function fn(e,t){let r=ye(e.metadata_json),n=It(`${e.path} ${e.title} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"wiki_page",id:e.id,title:e.title,text:null,score:0,scores:{catalog:n},source:null,citation:null,artifact:{uri:e.artifact_uri,path:e.path,hash:e.content_hash,shard_key:e.path},provenance:ze(r),reasons:["wiki_catalog_match"]};return i.score=se(i.scores,i.citation),i}function gn(e,t){let r=ye(e.metadata_json),n=It(`${e.kind} ${e.name} ${e.shard_key??""} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"knowledge_index",id:e.id,title:e.name,text:null,score:0,scores:{catalog:n},source:null,citation:null,artifact:{uri:e.artifact_uri,path:U(r,["artifact_key"]),hash:U(r,["content_hash"]),shard_key:e.shard_key},provenance:ze(r),reasons:["index_catalog_match"]};return i.score=se(i.scores,i.citation),i}function Ee(e,t){let r=`${t.kind}:${t.id}`,n=e.get(r);if(!n){e.set(r,t);return}n.scores={keyword:Math.max(n.scores.keyword??0,t.scores.keyword??0)||void 0,semantic:Math.max(n.scores.semantic??0,t.scores.semantic??0)||void 0,catalog:Math.max(n.scores.catalog??0,t.scores.catalog??0)||void 0},n.reasons=Nt([...n.reasons,...t.reasons]),n.text=n.text??t.text,n.title=n.title??t.title,n.source=n.source??t.source,n.citation=n.citation??t.citation,n.artifact=n.artifact??t.artifact,n.provenance=n.provenance??t.provenance,n.score=se(n.scores,n.citation)}function pn(e){let t={source_chunk:0,wiki_chunk:1,wiki_page:2,knowledge_index:3};return e.sort((r,n)=>{if(n.score!==r.score)return n.score-r.score;return t[r.kind]-t[n.kind]||r.id.localeCompare(n.id)})}async function be(e){let t=e.query.trim();if(!t)throw Error("Search query is required.");let r=Math.max(1,Math.min(e.limit??10,100)),n=rn(t),i=nn(n),s=e.semantic===!0||e.fake===!0||Boolean(e.modelRef),c=[],u=null,_=null,a=null,d=0,o=0,l=0,f=new Map;x(e.dbPath);let g=b(e.dbPath);try{let h=dn(g,i,Math.max(r*3,20));d=h.length,h.forEach((k,p)=>Ee(f,_n(k,on(k.rank,p))));let m=un(g,n,Math.max(r,10)),T=ln(g,n,Math.max(r,10));o=m.length+T.length,m.forEach((k)=>Ee(f,fn(k,n))),T.forEach((k)=>Ee(f,gn(k,n)))}finally{g.close()}if(s)try{let h=await he({dbPath:e.dbPath,query:t,limit:Math.max(r*3,20),config:e.config,env:e.env,modelRef:e.modelRef,dimensions:e.dimensions,fake:e.fake,batchSize:e.batchSize,maxParallelCalls:e.maxParallelCalls});u=h.provider,_=h.model,a=h.dimensions,l=h.results.length;for(let m of h.results){let T={kind:"source_chunk",id:m.chunk_id,title:null,text:m.text,score:0,scores:{semantic:an(m.score)},source:{uri:m.source_uri,ref:m.source_ref,kind:m.provenance?.source_kind??null,revision:m.revision,hash:m.hash},citation:{chunk_id:m.chunk_id,start_offset:m.provenance?.start_offset??null,end_offset:m.provenance?.end_offset??null},artifact:null,provenance:m.provenance,reasons:["semantic_match"]};T.score=se(T.scores,T.citation),Ee(f,T)}}catch(h){c.push(`semantic_search_failed: ${h instanceof Error?h.message:String(h)}`)}let E=pn(Array.from(f.values())).slice(0,r);return{query:t,limit:r,mode:{keyword:!0,catalog:!0,semantic:s},semantic_provider:u,semantic_model:_,semantic_dimensions:a,counts:{keyword_results:d,catalog_results:o,semantic_results:l,merged_results:E.length},warnings:c,results:E}}function Ct(e,t){return`${e}_${mn("sha256").update(t).digest("hex").slice(0,20)}`}function Pt(e){return e.normalize("NFKC").trim().replace(/\s+/g," ").toLowerCase()}function hn(e){return Array.from(new Set(Pt(e).match(/[\p{L}\p{N}_]+/gu)??[])).slice(0,16)}function En(e){return[e.title,e.text].filter(Boolean).join(" ").toLowerCase()}function yn(e,t){if(t.length===0)return 0;let r=En(e),n=t.filter((i)=>r.includes(i)).length;return Number((n/t.length).toFixed(6))}function kn(e){if(!e)return!0;if("read_only"in e)return e.read_only===!0;if("read_only_sources"in e)return e.read_only_sources===!0;return!0}function Dt(e){if(!e)return!1;if("stale"in e&&e.stale)return!0;if("status"in e)return $e(e.status);return!1}function bn(e){if(Dt(e.provenance))return 0;if(e.source?.hash||e.source?.revision)return 1;if(e.artifact?.hash)return 0.85;if(e.provenance&&"source_refs"in e.provenance&&e.provenance.source_refs.length>0)return 0.75;return 0.55}function vn(e){if(e.citation?.chunk_id&&(e.source?.uri||e.artifact?.uri))return 1;if(e.provenance&&"citation_required"in e.provenance&&e.provenance.citation_required)return 0.75;if(e.artifact?.uri)return 0.65;return 0.35}function wn(e){if(e.kind==="wiki_chunk")return 0.85;if(e.kind==="source_chunk")return 0.8;if(e.kind==="wiki_page")return 0.65;return 0.55}function Tn(e,t){let r={base_score:e.score,exact_score:yn(e,t),citation_score:vn(e),freshness_score:bn(e),authority_score:wn(e)},n=Math.min(1,r.base_score*0.65+r.exact_score*0.1+r.citation_score*0.1+r.freshness_score*0.1+r.authority_score*0.05),i=new Set(e.reasons);if(r.exact_score>0.5)i.add("exact_term");if(r.citation_score>=0.75)i.add("cited_source");if(r.freshness_score>=0.85)i.add("fresh_source");return{...e,score:Number(n.toFixed(6)),reasons:Array.from(i),rerank:{...r,final_score:Number(n.toFixed(6))}}}function Ut(e,t){let r=e.text??e.title;if(!r)return null;let n=r.replace(/\s+/g," ").trim();return n.length<=t?n:`${n.slice(0,Math.max(0,t-1)).trim()}...`}function xn(e){return{id:Ct("cite",`${e.kind}\x00${e.id}\x00${e.source?.uri??""}\x00${e.artifact?.uri??""}`),result_id:e.id,kind:e.kind,source_uri:e.source?.uri??null,source_ref:e.source?.ref??null,artifact_uri:e.artifact?.uri??null,artifact_path:e.artifact?.path??null,revision:e.source?.revision??null,hash:e.source?.hash??e.artifact?.hash??null,chunk_id:e.citation?.chunk_id??null,start_offset:e.citation?.start_offset??null,end_offset:e.citation?.end_offset??null,quote:Ut(e,500),provenance:e.provenance}}function Sn(e,t,r){let n=Ut(e,r);if(!n)return null;return{id:Ct("excerpt",`${e.kind}\x00${e.id}`),result_id:e.id,citation_id:t.id,kind:e.kind,text:n,score:e.score}}function ve(e){return e.map(()=>"?").join(", ")}function Rn(e,t){let r=t.map((u)=>u.citation?.chunk_id).filter((u)=>Boolean(u)),n=t.filter((u)=>u.kind==="wiki_page").map((u)=>u.id),i=[],s=[];if(r.length===0&&n.length===0)return{citations:i,backlinks:s};let c=b(e);try{if(r.length>0)i.push(...c.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
367
388
|
FROM citations
|
|
368
|
-
WHERE chunk_id IN (${
|
|
389
|
+
WHERE chunk_id IN (${ve(r)})
|
|
369
390
|
ORDER BY created_at DESC
|
|
370
391
|
LIMIT 50`).all(...r));if(n.length>0)i.push(...c.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
371
392
|
FROM citations
|
|
372
|
-
WHERE wiki_page_id IN (${
|
|
393
|
+
WHERE wiki_page_id IN (${ve(n)})
|
|
373
394
|
ORDER BY created_at DESC
|
|
374
395
|
LIMIT 50`).all(...n)),s.push(...c.query(`SELECT from_page_id, to_page_id, label
|
|
375
396
|
FROM wiki_backlinks
|
|
376
|
-
WHERE from_page_id IN (${
|
|
377
|
-
LIMIT 50`).all(...n,...n))}finally{c.close()}return{citations:i,backlinks:s}}async function
|
|
378
|
-
`)}function
|
|
397
|
+
WHERE from_page_id IN (${ve(n)}) OR to_page_id IN (${ve(n)})
|
|
398
|
+
LIMIT 50`).all(...n,...n))}finally{c.close()}return{citations:i,backlinks:s}}async function we(e){let t=Math.max(200,Math.min(e.contextChars??1200,4000)),r=await be(e),n=hn(r.query),i=[...r.warnings],s=new Set,c=new Set,_=r.results.filter((o)=>{if(!kn(o.provenance))return i.push(`permission_filtered: ${o.kind}:${o.id}`),s.add("Dropped a result because provenance was not read-only."),!1;if(Dt(o.provenance))return i.push(`stale_filtered: ${o.kind}:${o.id}`),c.add("Dropped a stale result whose source status requires reindexing."),!1;return!0}).map((o)=>Tn(o,n)).sort((o,l)=>l.score-o.score||o.id.localeCompare(l.id)).slice(0,r.limit),a=_.map(xn),d=_.map((o,l)=>Sn(o,a[l],t)).filter((o)=>Boolean(o));for(let o of _){if(o.provenance&&"read_only"in o.provenance&&o.provenance.read_only)s.add("All source-backed excerpts are read-only and citation-required.");if(o.rerank.freshness_score>=0.85)c.add("Fresh source revision/hash or artifact hash is present for top context.")}return{query:r.query,normalized_query:Pt(r.query),created_at:new Date().toISOString(),mode:r.mode,warnings:i,search_counts:r.counts,results:_,citations:a,excerpts:d,graph:Rn(e.dbPath,_),notes:{permissions:Array.from(s),freshness:Array.from(c)}}}function Je(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Ye(e){return`C${e+1}`}function On(e,t){if(t.excerpts.length===0)return`No indexed knowledge matched the prompt: ${e}`;return[`Found ${t.excerpts.length} relevant knowledge excerpt(s) for: ${e}`,"",...t.excerpts.slice(0,5).map((n,i)=>{let s=t.citations.find((u)=>u.id===n.citation_id),c=s?.source_ref??s?.source_uri??s?.artifact_path??s?.artifact_uri??"unknown source";return`[${Ye(i)}] ${n.text} (${c})`})].join(`
|
|
399
|
+
`)}function Nn(e,t){let r=t.citations.map((i,s)=>({id:Ye(s),source_ref:i.source_ref,source_uri:i.source_uri,artifact_path:i.artifact_path,revision:i.revision,hash:i.hash,quote:i.quote})),n=t.excerpts.map((i,s)=>({id:Ye(s),kind:i.kind,text:i.text,score:i.score}));return[`Prompt: ${e}`,"","Use only the provided context. Cite claims with citation ids like [C1]. If context is insufficient, say what is missing.","",`Context excerpts:
|
|
379
400
|
${JSON.stringify(n,null,2)}`,"",`Citations:
|
|
380
401
|
${JSON.stringify(r,null,2)}`].join(`
|
|
381
|
-
`)}function
|
|
382
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[t.runId,"knowledge-prompt",t.prompt,t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.now])}finally{r.close()}}function Ge(e,t){let r=
|
|
383
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${jt()}`,t.runId,t.level,t.event,JSON.stringify(t.metadata),t.now])}finally{r.close()}}function
|
|
402
|
+
`)}function An(e,t){if(t.citations.length===0)return[];return[{kind:"answer_note",title:e.length>80?`${e.slice(0,77)}...`:e,citations:t.citations.map((r)=>r.id),requires_approval:!0}]}function In(e,t){let r=b(e);try{r.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
403
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[t.runId,"knowledge-prompt",t.prompt,t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.now])}finally{r.close()}}function Ge(e,t){let r=b(e);try{r.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
404
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${jt()}`,t.runId,t.level,t.event,JSON.stringify(t.metadata),t.now])}finally{r.close()}}function Mt(e,t){let r=b(e);try{r.run(`UPDATE runs
|
|
384
405
|
SET status = ?, provider = ?, model = ?, metadata_json = ?, updated_at = ?
|
|
385
|
-
WHERE id = ?`,[t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.runId])}finally{r.close()}}function
|
|
406
|
+
WHERE id = ?`,[t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.runId])}finally{r.close()}}function Ln(e,t,r,n,i,s,c={}){let u=b(e);try{ge(u,{run_id:t,provider:n,model:i,input_tokens:r.input_tokens,output_tokens:r.output_tokens,cost_usd:r.cost_usd,metadata:c,created_at:s})}finally{u.close()}}async function Kt(e){let t=e.prompt.trim();if(!t)throw Error("Knowledge prompt is required.");let r=(e.now??new Date).toISOString(),n=`run_${jt()}`,i=X(e.modelRef??"default",e.config),s=L(i);x(e.dbPath),In(e.dbPath,{runId:n,prompt:t,status:e.generate?"running":"dry_run",provider:e.generate?s.provider:"local",model:e.generate?s.model:"context-draft",metadata:{semantic:e.semantic===!0||e.fake===!0||Boolean(e.modelRef),approve_write:e.approveWrite===!0,generated:e.generate===!0},now:r});let{prompt:c,generate:u,approveWrite:_,now:a,...d}=e,o=await we({...d,query:t});Ge(e.dbPath,{runId:n,level:"info",event:"context_retrieved",metadata:{results:o.results.length,citations:o.citations.length,warnings:o.warnings},now:r});let l=On(t,o),f=!1,g="local",E="context-draft",h={input_tokens:Je(t)+o.excerpts.reduce((p,R)=>p+Je(R.text),0),output_tokens:Je(l),cost_usd:0},m=[...o.warnings];if(e.generate)try{if(e.fake)f=!0,g=s.provider,E=s.model,l=`Fake generated answer for: ${t}
|
|
386
407
|
|
|
387
|
-
${l}`;else{let{generateText:p}=await import("ai"),
|
|
388
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function
|
|
389
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${
|
|
390
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function
|
|
408
|
+
${l}`;else{let{generateText:p}=await import("ai"),R=await kt(i,{config:e.config,env:e.env}),N=await p({model:R,system:"You answer company knowledge-base prompts using only provided context and citation ids.",prompt:Nn(t,o)});f=!0,g=s.provider,E=s.model,l=N.text;let v=fe({provider:g,model:E,usage:N.usage,providerMetadata:N.providerMetadata});h={input_tokens:v.input_tokens,output_tokens:v.output_tokens,cost_usd:v.cost_usd}}}catch(p){throw Ge(e.dbPath,{runId:n,level:"error",event:"answer_generation_failed",metadata:{message:p instanceof Error?p.message:String(p)},now:r}),Mt(e.dbPath,{runId:n,status:"failed",provider:s.provider,model:s.model,metadata:{generated:!1,error:p instanceof Error?p.message:String(p)},now:r}),p}let T=An(t,o),k={approved:e.approveWrite===!0,durable_writes_performed:!1,reason:e.approveWrite?"Approval flag recorded; durable wiki writing is deferred to the wiki compile task.":"Dry-run mode: proposed wiki updates require approval before durable writes."};return Ge(e.dbPath,{runId:n,level:"info",event:f?"answer_generated":"answer_drafted",metadata:{provider:g,model:E,proposed_updates:T.length,durable_writes_performed:!1},now:r}),Ln(e.dbPath,n,h,g,E,r,{generated:f,citations:o.citations.length}),Mt(e.dbPath,{runId:n,status:f?"completed":"dry_run",provider:g,model:E,metadata:{generated:f,citations:o.citations.length,proposed_updates:T.length,approve_write:e.approveWrite===!0},now:r}),{run_id:n,prompt:t,generated:f,provider:g,model:E,answer:l,context:o,citations:o.citations,proposed_wiki_updates:T,write_policy:k,usage:h,warnings:m}}import{createHash as qn,randomUUID as Hn}from"crypto";import{existsSync as Bn,readFileSync as zn}from"fs";import{basename as Jn}from"path";function Ft(e,t){if(!e)throw Error(t);return e}function Cn(e){let r=e.slice(13).split("/").filter(Boolean),n=r[0];if(n!=="file"&&n!=="source")throw Error("Invalid open-files ref. Expected open-files://file/<id>, open-files://file/<id>/revision/<revision_id>, or open-files://source/<id>/path/<path>.");let i=Ft(r[1],"Invalid open-files ref. Missing id.");if(n==="file"){if(r.length===2)return{kind:"open-files",uri:e,entity:n,id:i};if(r[2]==="revision"&&r[3]&&r.length===4)return{kind:"open-files",uri:e,entity:n,id:i,revision_id:decodeURIComponent(r[3])};throw Error("Invalid open-files file ref. Expected open-files://file/<id>/revision/<revision_id>.")}let s=r.indexOf("path"),c=s>=0?decodeURIComponent(r.slice(s+1).join("/")):void 0;return{kind:"open-files",uri:e,entity:n,id:i,path:c}}function Pn(e){let t=new URL(e),r=Ft(t.hostname,"Invalid s3 ref. Missing bucket."),n=decodeURIComponent(t.pathname.replace(/^\/+/,""));if(!n)throw Error("Invalid s3 ref. Missing object key.");return{kind:"s3",uri:e,bucket:r,key:n}}function Dn(e){let t=new URL(e);return{kind:"file",uri:e,path:decodeURIComponent(t.pathname)}}function Un(e){let t=new URL(e);return{kind:"web",uri:e,url:t.toString()}}function M(e){if(e.startsWith("open-files://"))return Cn(e);if(e.startsWith("s3://"))return Pn(e);if(e.startsWith("file://"))return Dn(e);if(e.startsWith("https://")||e.startsWith("http://"))return Un(e);throw Error(`Unsupported source ref scheme: ${e}`)}function Wt(e,t=M(e)){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Xt(e){let t=M(e);return t.kind==="open-files"&&t.entity==="file"?t.revision_id??null:null}import{createHash as Mn,randomUUID as Ve}from"crypto";import{relative as jn,resolve as qt,sep as Kn}from"path";function $t(e){let t=process.env[e];return t==="1"||t==="true"||t==="yes"}function Ht(e,t){let r=e,n=new Set(r.safety?.network?.allowed_s3_buckets??[]);if(e.storage.type==="s3"&&e.storage.s3?.bucket)n.add(e.storage.s3.bucket);if(process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS)for(let i of process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.split(",").map((s)=>s.trim()).filter(Boolean))n.add(i);return{mode:e.mode,allowWriteRoots:[t.home,t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir].map((i)=>qt(i)),readOnlySourceAccess:!0,network:{webSearchEnabled:r.safety?.network?.web_search_enabled??$t("HASNA_KNOWLEDGE_WEB_SEARCH"),s3ReadsEnabled:r.safety?.network?.s3_reads_enabled??$t("HASNA_KNOWLEDGE_ALLOW_S3_READS"),allowedS3Buckets:[...n].sort()},redaction:{enabled:r.safety?.redaction?.enabled??!0},approvals:{generatedWritesRequireApproval:r.safety?.approvals?.generated_writes_require_approval??!0}}}function Fn(e,t){let r=jn(e,t);return r===""||!r.startsWith("..")&&r!==".."&&!r.startsWith(`..${Kn}`)}function J(e,t){let r=qt(e);if(!t.allowWriteRoots.some((n)=>Fn(n,r)))throw Error(`Safety policy denied write outside .hasna/apps/knowledge: ${e}`)}function q(e,t){let n=new URL(e).hostname;if(!t.network.s3ReadsEnabled)throw Error("Safety policy denied S3 read. Set safety.network.s3_reads_enabled=true or HASNA_KNOWLEDGE_ALLOW_S3_READS=1.");if(!t.network.allowedS3Buckets.includes(n))throw Error(`Safety policy denied S3 bucket "${n}". Add it to safety.network.allowed_s3_buckets or HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.`)}function Q(e){if(!e.network.webSearchEnabled)throw Error("Safety policy denied web search. Set safety.network.web_search_enabled=true or HASNA_KNOWLEDGE_WEB_SEARCH=1.")}var Wn=[{type:"private_key_block",severity:"high",regex:/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,replacement:"[REDACTED:private_key_block]"},{type:"secret_assignment",severity:"high",regex:/\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*['"]?[^'"\s]{8,}/gi,replacement:"[REDACTED:secret_assignment]"},{type:"openai_api_key",severity:"high",regex:/\bsk-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:openai_api_key]"},{type:"anthropic_api_key",severity:"high",regex:/\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:anthropic_api_key]"},{type:"aws_access_key_id",severity:"high",regex:/\bA(?:KIA|SIA)[A-Z0-9]{16}\b/g,replacement:"[REDACTED:aws_access_key_id]"}];function Te(e,t){if(t&&!t.redaction.enabled)return{text:e,findings:[]};let r=e,n=[];for(let i of Wn)r=r.replace(i.regex,(s,...c)=>{let u=typeof c.at(-2)==="number"?c.at(-2):r.indexOf(s);return n.push({type:i.type,severity:i.severity,start:Math.max(0,u),end:Math.max(0,u+s.length)}),i.replacement});return{text:r,findings:n}}function Xn(e){return`audit_${Mn("sha256").update(`${e.event_type}\x00${e.action}\x00${e.target_uri??""}\x00${e.created_at??""}\x00${JSON.stringify(e.metadata??{})}\x00${Ve()}`).digest("hex").slice(0,24)}`}function A(e,t){let r=t.created_at??new Date().toISOString(),n=Xn({...t,created_at:r});return e.run(`INSERT INTO audit_events (id, event_type, action, target_uri, decision, metadata_json, created_at)
|
|
409
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function xe(e,t){let r=t.created_at??new Date().toISOString();for(let n of t.findings)e.run(`INSERT INTO redaction_findings (id, source_uri, run_id, severity, finding_type, metadata_json, created_at)
|
|
410
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${Ve()}`,t.source_uri??null,t.run_id??null,n.severity,n.type,JSON.stringify({...t.metadata??{},start:n.start,end:n.end}),r]);return t.findings.length}function Bt(e,t){let r=t.created_at??new Date().toISOString(),n=`approval_${Ve()}`;return e.run(`INSERT INTO approval_gates (id, action, target_uri, status, reason, approved_by, metadata_json, created_at, updated_at)
|
|
411
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function $n(e,t,r){let n=e.query(`SELECT id FROM approval_gates
|
|
391
412
|
WHERE action = ? AND status = 'approved' AND (target_uri IS NULL OR target_uri = ? OR ? IS NULL)
|
|
392
|
-
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function
|
|
413
|
+
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function zt(e,t,r,n){let i=r==="generated_write"&&t.approvals.generatedWritesRequireApproval,s=!i||$n(e,r,n);return{action:r,target_uri:n??null,approval_required:i,approved:s,decision:s?"allow":"requires_approval"}}function Se(e,t){return`${e}_${qn("sha256").update(t).digest("hex").slice(0,20)}`}function Z(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function S(e){return typeof e==="string"&&e.length>0?e:void 0}function Gn(e){let t=S(e.source_ref)??S(e.source_uri)??S(e.uri);if(t)return t;let r=S(e.file_id);if(r){let s=S(e.revision_id)??S(e.revision),c=`open-files://file/${encodeURIComponent(r)}`;return s?`${c}/revision/${encodeURIComponent(s)}`:c}let n=S(e.source_id),i=S(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Outbox event is missing source_ref, file_id, or source_id/path.")}function Yn(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Vn(e){return S(e.hash)??S(e.checksum)??S(e.sha256)??null}function Qn(e,t,r){return S(e.revision_id)??S(e.revision)??S(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??null}function Zn(e){return(S(e.event)??S(e.type)??S(e.action)??S(e.change_type)??"changed").toLowerCase()}function ei(e){let t=S(e.path);return S(e.title)??S(e.name)??(t?Jn(t):null)}function ti(e,t){let r=Gn(e),n=M(r),i=Vn(e);return{raw:e,eventType:Zn(e),sourceRef:r,sourceUri:Yn(r,n),kind:n.kind,title:ei(e),revision:Qn(e,n,i),hash:i,status:S(e.status)?.toLowerCase()??null,updatedAt:S(e.updated_at)??t,acl:e.permissions??e.acl??void 0}}function ri(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Outbox array parse failed.");return r.map((n)=>{let i=Z(n);if(!i)throw Error("Outbox array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=Z(r);if(!n)throw Error("Outbox object parse failed.");if(Array.isArray(n.events))return n.events.map((i)=>{let s=Z(i);if(!s)throw Error("Outbox events entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=Z(JSON.parse(i));if(!s)throw Error("Outbox JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=Z(JSON.parse(r));if(!n)throw Error("Outbox JSONL entries must be objects.");return n})}async function ni(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 outbox URI: ${e}`);if(r)q(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function ii(e,t,r){if(e.startsWith("s3://"))return ni(e,t,r);if(!Bn(e))throw Error(`Outbox not found: ${e}`);return zn(e,"utf8")}function Jt(e,t){let r={};if(e)try{r=Z(JSON.parse(e))??{}}catch{r={}}return JSON.stringify({...r,...t})}function si(e,t,r){let n=Se("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
393
414
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
394
415
|
ON CONFLICT(uri) DO UPDATE SET
|
|
395
416
|
kind = excluded.kind,
|
|
396
417
|
title = COALESCE(excluded.title, sources.title),
|
|
397
|
-
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(
|
|
418
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(S(t.raw.path))s.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[Jt(i.metadata_json,s),t.acl===void 0?null:JSON.stringify(t.acl),t.acl===void 0?null:JSON.stringify(t.acl),t.updatedAt,i.id]),i.id}function oi(e,t,r,n){if(!r.revision)return null;let i=Se("rev",`${t}\x00${r.revision}`),s={source_ref:r.sourceRef,source_uri:r.sourceUri,status:r.status,last_outbox_event:r.eventType,reindex_required:!0};return e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
398
419
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
399
420
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
400
421
|
hash = COALESCE(excluded.hash, source_revisions.hash),
|
|
401
|
-
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,
|
|
402
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:n.length}),t,t]);let c=new Set,u=new Set,_=0,a=0,d=0,o=0,l=0,f=0,g=0;return A(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:n.length,read_only:!0},created_at:t}),n.forEach((
|
|
422
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,S(r.raw.extracted_text_ref)??null,JSON.stringify(s),n]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision)?.id??null}function ai(e,t,r){if(r.revision)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").all(t,r.revision).map((n)=>n.id);if(r.hash)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND hash = ?").all(t,r.hash).map((n)=>n.id);return e.query("SELECT id FROM source_revisions WHERE source_id = ?").all(t).map((n)=>n.id)}function ci(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t),n=0,i=0;for(let c of r){let u=e.query("SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?").get(c.id);n+=u?.n??0;let _=e.query("SELECT COUNT(*) AS n FROM vector_index_entries WHERE chunk_id = ?").get(c.id);i+=_?.n??0,e.run("DELETE FROM vector_index_entries WHERE chunk_id = ?",[c.id]),e.run("DELETE FROM chunk_embeddings WHERE chunk_id = ?",[c.id]),e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[c.id])}e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]);let s=e.query("SELECT metadata_json FROM source_revisions WHERE id = ?").get(t);return e.run("UPDATE source_revisions SET metadata_json = ? WHERE id = ?",[Jt(s?.metadata_json,{reindex_required:!0,invalidated_at:new Date().toISOString()}),t]),{chunksDeleted:r.length,embeddingsDeleted:n,vectorEntriesDeleted:i}}function di(e,t){return t==="deleted"||["delete","deleted","remove","removed"].includes(e)}function ui(e){return["move","moved","rename","renamed","path_changed"].includes(e)}function li(e){return["permission","permissions","permission_changed","acl_changed"].includes(e)}async function Gt(e){let t=(e.now??new Date).toISOString();if(e.safetyPolicy)J(e.dbPath,e.safetyPolicy);x(e.dbPath);let r=await ii(e.input,e.config,e.safetyPolicy),n=ri(r),i=b(e.dbPath),s=`run_${Hn()}`;try{return i.transaction(()=>{i.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
423
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:n.length}),t,t]);let c=new Set,u=new Set,_=0,a=0,d=0,o=0,l=0,f=0,g=0;return A(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:n.length,read_only:!0},created_at:t}),n.forEach((E,h)=>{let m=ti(E,t),T=si(i,m,t);c.add(T);let k=oi(i,T,m,t);if(k)u.add(k);let p=ai(i,T,m);for(let R of p){u.add(R);let N=ci(i,R);_+=N.chunksDeleted,a+=N.embeddingsDeleted,d+=N.vectorEntriesDeleted,o+=1}if(di(m.eventType,m.status))l+=1;if(ui(m.eventType))f+=1;if(li(m.eventType)||m.acl!==void 0)g+=1;i.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
403
424
|
VALUES (?, ?, ?, ?, ?, ?)`,[Se("evt",`${s}\x00${h}\x00${m.sourceRef}\x00${m.eventType}`),s,"info",m.eventType,JSON.stringify({source_ref:m.sourceRef,source_uri:m.sourceUri,revision:m.revision,hash:m.hash,status:m.status,affected_revisions:p.length}),m.updatedAt])}),i.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
404
|
-
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[Se("usage",s),s,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),A(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:s,events:n.length,sources:c.size,revisions:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:s,events_seen:n.length,sources_touched:c.size,revisions_touched:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d,stale_revisions:o,deleted_sources:l,moved_sources:f,permission_updates:g}})()}finally{i.close()}}import{createHash as
|
|
425
|
+
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[Se("usage",s),s,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),A(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:s,events:n.length,sources:c.size,revisions:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:s,events_seen:n.length,sources_touched:c.size,revisions_touched:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d,stale_revisions:o,deleted_sources:l,moved_sources:f,permission_updates:g}})()}finally{i.close()}}import{createHash as _i}from"crypto";import{existsSync as fi,readFileSync as gi}from"fs";import{basename as pi}from"path";function Qe(e,t){return`${e}_${_i("sha256").update(t).digest("hex").slice(0,20)}`}function ee(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function w(e){return typeof e==="string"&&e.length>0?e:void 0}function mi(e){return typeof e==="number"&&Number.isFinite(e)?e:void 0}function hi(e){let t=w(e.source_ref)??w(e.source_uri)??w(e.uri);if(t)return t;let r=w(e.file_id);if(r){let s=w(e.revision_id)??w(e.revision),c=`open-files://file/${encodeURIComponent(r)}`;return s?`${c}/revision/${encodeURIComponent(s)}`:c}let n=w(e.source_id),i=w(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Manifest item is missing source_ref, file_id, or source_id/path.")}function Ei(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function yi(e){let t=w(e.extracted_text)??w(e.text)??w(e.content_text)??w(e.markdown);if(t!==void 0)return t;let r=e.content;return typeof r==="string"?r:null}function ki(e){let t=w(e.extracted_text_ref)??w(e.extracted_text_uri)??w(e.text_ref);if(t)return t;let r=ee(e.content);return w(r?.extracted_text_ref)??w(r?.extracted_text_uri)??null}function bi(e){let t=w(e.path);return w(e.title)??w(e.name)??(t?pi(t):null)}function vi(e){return w(e.hash)??w(e.checksum)??w(e.sha256)??null}function wi(e,t,r){return w(e.revision_id)??w(e.revision)??w(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??w(e.updated_at)??"current"}function Ti(e,t){let r={};for(let[n,i]of Object.entries(e)){if(["text","content","content_text","extracted_text","markdown"].includes(n))continue;r[n]=i}return r.source_ref=t.sourceRef,r.source_uri=t.sourceUri,r.status=t.status,r}function xi(e,t){let r=hi(e),n=M(r),i=Ei(r,n),s=vi(e),c=w(e.status)??"active";return{raw:e,sourceRef:r,sourceUri:i,kind:n.kind,title:bi(e),revision:wi(e,n,s),hash:s,extractedTextUri:ki(e),text:yi(e),metadata:Ti(e,{sourceRef:r,sourceUri:i,status:c}),acl:e.permissions??e.acl??{},status:c,updatedAt:w(e.updated_at)??t}}function Si(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Manifest array parse failed.");return r.map((n)=>{let i=ee(n);if(!i)throw Error("Manifest array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=ee(r);if(!n)throw Error("Manifest object parse failed.");if(Array.isArray(n.items))return n.items.map((i)=>{let s=ee(i);if(!s)throw Error("Manifest items entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=ee(JSON.parse(i));if(!s)throw Error("Manifest JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=ee(JSON.parse(r));if(!n)throw Error("Manifest JSONL entries must be objects.");return n})}async function Ri(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 manifest URI: ${e}`);if(r)q(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function Oi(e,t,r){if(e.startsWith("s3://"))return Ri(e,t,r);if(!fi(e))throw Error(`Manifest not found: ${e}`);return gi(e,"utf8")}function Ni(e,t,r){let n=e.replace(/\r\n/g,`
|
|
405
426
|
`);if(!n.trim())return[];let i=[],s=0;while(s<n.length){let c=Math.min(n.length,s+t),u=c;if(c<n.length){let a=n.lastIndexOf(`
|
|
406
427
|
|
|
407
|
-
`,c),d=n.lastIndexOf(". ",c),o=Math.max(a,d);if(o>s+Math.floor(t*0.5))u=o+(o===a?2:1)}let _=n.slice(s,u).trim();if(_)i.push({ordinal:i.length,text:_,startOffset:s,endOffset:u});if(u>=n.length)break;s=Math.max(0,u-r)}return i}function
|
|
428
|
+
`,c),d=n.lastIndexOf(". ",c),o=Math.max(a,d);if(o>s+Math.floor(t*0.5))u=o+(o===a?2:1)}let _=n.slice(s,u).trim();if(_)i.push({ordinal:i.length,text:_,startOffset:s,endOffset:u});if(u>=n.length)break;s=Math.max(0,u-r)}return i}function Ai(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Ii(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t);for(let n of r)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[n.id]);return e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]),r.length}function Li(e,t,r){let n=Qe("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
408
429
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
409
430
|
ON CONFLICT(uri) DO UPDATE SET
|
|
410
431
|
kind = excluded.kind,
|
|
411
432
|
title = excluded.title,
|
|
412
433
|
metadata_json = excluded.metadata_json,
|
|
413
434
|
acl_json = excluded.acl_json,
|
|
414
|
-
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function
|
|
435
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function Ci(e,t,r,n){let i=Qe("rev",`${t}\x00${r.revision}`);e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
415
436
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
416
437
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
417
438
|
hash = excluded.hash,
|
|
418
439
|
extracted_text_uri = excluded.extracted_text_uri,
|
|
419
|
-
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision);if(!s)throw Error(`Failed to upsert source revision: ${r.sourceRef}`);return s.id}function
|
|
420
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[d,t,"source",a.ordinal,a.text,
|
|
440
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision);if(!s)throw Error(`Failed to upsert source revision: ${r.sourceRef}`);return s.id}function Pi(e,t,r,n,i,s,c){if(!r.text||r.status.toLowerCase()==="deleted")return{chunksInserted:0,redactions:0};let u=Te(r.text,c);if(u.findings.length>0)xe(e,{source_uri:r.sourceUri,findings:u.findings,metadata:{source_ref:r.sourceRef,revision:r.revision},created_at:n}),A(e,{event_type:"redaction",action:"source_text_redact",target_uri:r.sourceUri,decision:"redacted",metadata:{findings:u.findings.length,source_ref:r.sourceRef,revision:r.revision},created_at:n});let _=Ni(u.text,i,s);for(let a of _){let d=Qe("chk",`${t}\x00${a.ordinal}\x00${a.text}`),o=$({source_ref:r.sourceRef,source_uri:r.sourceUri,source_kind:r.kind,source_revision_id:t,revision:r.revision,hash:r.hash,chunk_id:d,start_offset:a.startOffset,end_offset:a.endOffset,status:r.status,resolver:"open-files-read-only"}),l=bt({source_ref:r.sourceRef,source_uri:r.sourceUri,source_kind:r.kind,source_revision_id:t,revision:r.revision,hash:r.hash,status:r.status,path:w(r.raw.path)??null,mime:w(r.raw.mime)??w(r.raw.content_type)??null,size:mi(r.raw.size)??null},o);e.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
441
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[d,t,"source",a.ordinal,a.text,Ai(a.text),a.startOffset,a.endOffset,JSON.stringify(l),n]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[d,a.text,r.title??"",r.sourceUri])}return{chunksInserted:_.length,redactions:u.findings.length}}async function Yt(e){let t=e.now??new Date;if(e.safetyPolicy)J(e.dbPath,e.safetyPolicy);x(e.dbPath);let r=await Oi(e.input,e.config,e.safetyPolicy),n=Si(r);return oe({dbPath:e.dbPath,items:n,sourceLabel:e.input,safetyPolicy:e.safetyPolicy,now:t,maxChunkChars:e.maxChunkChars,chunkOverlapChars:e.chunkOverlapChars})}async function oe(e){let t=(e.now??new Date).toISOString(),r=e.maxChunkChars??4000,n=e.chunkOverlapChars??200;if(r<500)throw Error("maxChunkChars must be at least 500.");if(n<0||n>=r)throw Error("chunkOverlapChars must be less than maxChunkChars.");if(e.safetyPolicy)J(e.dbPath,e.safetyPolicy);x(e.dbPath);let i=b(e.dbPath);try{return i.transaction(()=>{let c=new Set,u=new Set,_=0,a=0,d=0,o=0;A(i,{event_type:"source_read",action:e.readAction??(e.sourceLabel.startsWith("s3://")?"s3_manifest_read":"local_manifest_read"),target_uri:e.sourceLabel,decision:"allow",metadata:{items:e.items.length,read_only:!0},created_at:t});for(let l of e.items){let f=xi(l,t),g=Li(i,f,t),E=Ci(i,g,f,t);if(c.add(g),u.add(E),f.text||f.status.toLowerCase()==="deleted")a+=Ii(i,E);let h=Pi(i,E,f,t,r,n,e.safetyPolicy);_+=h.chunksInserted,d+=h.redactions}return A(i,{event_type:"write",action:"knowledge_manifest_ingest",target_uri:e.dbPath,decision:"allow",metadata:{items:e.items.length,sources:c.size,revisions:u.size,chunks_inserted:_,redactions:d},created_at:t}),{path:e.sourceLabel,db_path:e.dbPath,items_seen:e.items.length,sources_upserted:c.size,revisions_upserted:u.size,chunks_inserted:_,chunks_deleted:a,redactions:d,skipped:o}})()}finally{i.close()}}import{createHash as Wi}from"crypto";import{existsSync as Xi,readFileSync as $i}from"fs";import{basename as Ne}from"path";function Re(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function G(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function Vt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Di(e,t){let r=e.mode;if(typeof r==="string"&&r!=="read_only")throw Error(`Source resolver denied ${t}. Permission mode is ${r}, expected read_only.`);let n=e.denied_purposes;if(Array.isArray(n)&&n.includes(t))throw Error(`Source resolver denied ${t}. Purpose is explicitly denied.`);let i=e.allowed_purposes;if(Array.isArray(i)&&i.length>0&&!i.includes(t))throw Error(`Source resolver denied ${t}. Allowed purposes: ${i.join(", ")}`)}function Ui(e,t,r){if(!t)return r;try{let n=M(e);if(n.kind==="open-files"&&n.entity==="file")return`${e}/revision/${encodeURIComponent(t.revision)}`}catch{return r}return r}function Mi(e,t,r){return e.query(`SELECT id, uri, kind, title, metadata_json, acl_json, updated_at
|
|
421
442
|
FROM sources
|
|
422
443
|
WHERE uri = ? OR uri = ?
|
|
423
444
|
ORDER BY CASE WHEN uri = ? THEN 0 ELSE 1 END
|
|
424
|
-
LIMIT 1`).get(t,r,t)??null}function
|
|
445
|
+
LIMIT 1`).get(t,r,t)??null}function ji(e,t,r){if(r)return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
425
446
|
FROM source_revisions
|
|
426
447
|
WHERE source_id = ? AND revision = ?
|
|
427
448
|
LIMIT 1`).get(t,r)??null;return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
428
449
|
FROM source_revisions
|
|
429
450
|
WHERE source_id = ?
|
|
430
451
|
ORDER BY created_at DESC, revision DESC
|
|
431
|
-
LIMIT 1`).get(t)??null}function
|
|
452
|
+
LIMIT 1`).get(t)??null}function Ki(e,t){if(!t)return 0;return e.query("SELECT COUNT(*) AS n FROM chunks WHERE source_revision_id = ?").get(t)?.n??0}function Fi(e,t,r){if(!t||r<=0)return[];return e.query(`SELECT id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json
|
|
432
453
|
FROM chunks
|
|
433
454
|
WHERE source_revision_id = ?
|
|
434
455
|
ORDER BY ordinal ASC
|
|
435
|
-
LIMIT ?`).all(t,r)}async function
|
|
456
|
+
LIMIT ?`).all(t,r)}async function Oe(e){let t=e.purpose??"knowledge_answer",r=Math.max(0,Math.min(e.limit??10,100)),n=(e.now??new Date).toISOString(),i=M(e.sourceRef),s=Wt(e.sourceRef,i),c=Xt(e.sourceRef);if(e.safetyPolicy){if(!e.safetyPolicy.readOnlySourceAccess)throw Error("Safety policy denied source resolution.");J(e.dbPath,e.safetyPolicy)}x(e.dbPath);let u=b(e.dbPath);try{return u.transaction(()=>{let _=Mi(u,s,e.sourceRef);if(!_)return A(u,{event_type:"source_read",action:"open_files_resolve_missing",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:s},created_at:n}),{source_ref:e.sourceRef,source_uri:s,purpose:t,read_only:!0,resolved:!1,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:null,revision:null,content:{mime:null,size:null,hash:null,text_available:!1,chunks_total:0,chunks_returned:0,char_count_returned:0,extracted_text_ref:null,bytes_available:!1,bytes_exposed:!1},chunks:[],citations:[]};let a=Re(_.metadata_json),d=Re(_.acl_json);try{Di(d,t)}catch(p){throw A(u,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"deny",metadata:{purpose:t,read_only:!0,source_uri:_.uri,error:p instanceof Error?p.message:String(p)},created_at:n}),p}let o=ji(u,_.id,c),l=Re(o?.metadata_json),f=Ki(u,o?.id??null),g=Fi(u,o?.id??null,r),E=Ui(_.uri,o,e.sourceRef),h=g.map((p)=>{let R=Re(p.metadata_json),N={resolver:"open-files-read-only",mode:"local_catalog",purpose:t,read_only:!0,source_ref:G(R,["source_ref"])??E,source_uri:_.uri,source_revision_id:o?.id??null,revision:o?.revision??null,hash:o?.hash??G(R,["hash"]),chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,resolved_at:n},v=$({source_ref:N.source_ref,source_uri:N.source_uri,source_kind:_.kind,source_revision_id:N.source_revision_id,revision:N.revision,hash:N.hash,chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,status:G(R,["status"]),resolver:N.resolver});return{id:p.id,kind:p.kind,ordinal:p.ordinal,text:p.text,token_count:p.token_count,start_offset:p.start_offset,end_offset:p.end_offset,metadata:R,evidence:N,provenance:v}}),m=h.map((p)=>({source_ref:p.evidence.source_ref,source_uri:_.uri,chunk_id:p.id,quote:p.text.slice(0,500),start_offset:p.start_offset,end_offset:p.end_offset,evidence:p.evidence,provenance:p.provenance}));A(u,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:_.uri,revision:o?.revision??null,chunks_returned:h.length,chunks_total:f},created_at:n});let T=G(a,["mime","content_type"])??G(l,["mime","content_type"]),k=Vt(a,["size","size_bytes"])??Vt(l,["size","size_bytes"]);return{source_ref:E,source_uri:_.uri,purpose:t,read_only:!0,resolved:!0,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:{id:_.id,uri:_.uri,kind:_.kind,title:_.title,metadata:a,permissions:d,updated_at:_.updated_at},revision:o?{id:o.id,revision:o.revision,hash:o.hash,extracted_text_uri:o.extracted_text_uri,metadata:l,created_at:o.created_at,reindex_required:l.reindex_required===!0}:null,content:{mime:T,size:k,hash:o?.hash??G(a,["hash","checksum","sha256"]),text_available:f>0,chunks_total:f,chunks_returned:h.length,char_count_returned:h.reduce((p,R)=>p+R.text.length,0),extracted_text_ref:o?.extracted_text_uri??G(l,["extracted_text_ref","extracted_text_uri"]),bytes_available:!1,bytes_exposed:!1},chunks:h,citations:m}})()}finally{u.close()}}function te(e){return`sha256:${Wi("sha256").update(e).digest("hex")}`}function qi(e){return e.replace(/<script[\s\S]*?<\/script>/gi," ").replace(/<style[\s\S]*?<\/style>/gi," ").replace(/<[^>]+>/g," ").replace(/ /g," ").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/\s+\n/g,`
|
|
436
457
|
`).replace(/\n\s+/g,`
|
|
437
|
-
`).replace(/[ \t]{2,}/g," ").trim()}async function
|
|
438
|
-
|
|
439
|
-
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
458
|
+
`).replace(/[ \t]{2,}/g," ").trim()}async function Hi(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 source URI: ${e}`);if(r)q(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function Bi(e,t){if(t)Q(t);let r=await fetch(e,{headers:{accept:"text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5","user-agent":"@hasna/knowledge source-ingest"}});if(!r.ok)throw Error(`Web source read failed ${r.status}: ${e}`);let n=r.headers.get("content-type"),i=await r.text();return{text:n?.includes("html")?qi(i):i,mime:n}}function Ae(e){if(e.kind==="file")return Ne(e.path);if(e.kind==="s3")return Ne(e.key);if(e.kind==="web")return Ne(new URL(e.url).pathname)||e.url;return e.path?Ne(e.path):e.id}async function Qt(e,t,r){if(e.kind==="file"){if(!Xi(e.path))throw Error(`Source file not found: ${e.path}`);let n=$i(e.path,"utf8");return{text:n,contentSource:"file",title:Ae(e),mime:"text/plain",size:n.length,hash:te(n),revision:null,extractedTextRef:null,metadata:{path:e.path},permissions:{mode:"read_only"}}}if(e.kind==="s3"){let n=await Hi(e.uri,t,r);return{text:n,contentSource:"s3",title:Ae(e),mime:"text/plain",size:n.length,hash:te(n),revision:null,extractedTextRef:null,metadata:{bucket:e.bucket,key:e.key},permissions:{mode:"read_only"}}}if(e.kind==="web"){let n=await Bi(e.url,r);return{text:n.text,contentSource:"web",title:Ae(e),mime:n.mime,size:n.text.length,hash:te(n.text),revision:null,extractedTextRef:null,metadata:{url:e.url},permissions:{mode:"read_only"}}}throw Error(`Direct source reading is not available for ${e.uri}`)}async function zi(e,t,r){if(e.startsWith("open-files://"))throw Error("Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.");let n=M(e);return{text:(await Qt(n,t,r)).text,contentSource:"extracted_text_ref"}}async function Ji(e){let t=await Oe({dbPath:e.dbPath,sourceRef:e.sourceRef,purpose:e.purpose??"knowledge_index",limit:100,safetyPolicy:e.safetyPolicy,now:e.now});if(!t.resolved)throw Error("Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.");if(t.revision?.extracted_text_uri&&!t.content.text_available){let n=await zi(t.revision.extracted_text_uri,e.config,e.safetyPolicy);return{text:n.text,contentSource:n.contentSource,title:t.source?.title??null,mime:t.content.mime,size:n.text.length,hash:t.revision.hash??te(n.text),revision:t.revision.revision,extractedTextRef:t.revision.extracted_text_uri,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}if(t.chunks.length===0)throw Error("Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.");let r=t.chunks.map((n)=>n.text).join(`
|
|
459
|
+
|
|
460
|
+
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??te(r),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function Gi(e,t,r,n){let i=r.hash??te(r.text),s={...r.metadata,source_ref:e,content_source:r.contentSource,read_only:!0},c={source_ref:e,name:r.title??Ae(t),mime:r.mime??"text/plain",size:r.size??r.text.length,hash:i,revision:r.revision??i,status:"active",updated_at:new Date().toISOString(),permissions:{mode:"read_only",allowed_purposes:[n],...r.permissions},metadata:s,extracted_text_ref:r.extractedTextRef,extracted_text:r.text};if(t.kind==="open-files"){if(t.entity==="file")c.file_id=t.id;if(t.entity==="source")c.source_id=t.id,c.path=t.path}if(t.kind==="file")c.path=t.path;if(t.kind==="s3")c.path=t.key;if(t.kind==="web")c.url=t.url;return c}async function Zt(e){let t=e.purpose??"knowledge_index",r=M(e.sourceRef),n=r.kind==="open-files"?await Ji(e):await Qt(r,e.config,e.safetyPolicy),i=Gi(e.sourceRef,r,n,t);return{...await oe({dbPath:e.dbPath,items:[i],sourceLabel:e.sourceRef,readAction:"source_ref_ingest_read",safetyPolicy:e.safetyPolicy,now:e.now}),source_ref:e.sourceRef,content_source:n.contentSource,read_only:!0,hash:String(i.hash)}}import{createHash as Yi,randomUUID as er}from"crypto";function Vi(e,t){return`${e}_${Yi("sha256").update(t).digest("hex").slice(0,20)}`}function Qi(e){let t=b(e);try{let r=t.query("SELECT status, COUNT(*) AS n FROM reindex_queue GROUP BY status ORDER BY status").all();return Object.fromEntries(r.map((n)=>[n.status,n.n]))}finally{t.close()}}function tr(e,t){let r=z(t.modelRef,t.config),n=L(r),i=b(e);try{return i.query(`SELECT c.id AS chunk_id, c.source_revision_id, s.uri AS source_uri
|
|
461
|
+
FROM chunks c
|
|
462
|
+
LEFT JOIN source_revisions sr ON sr.id = c.source_revision_id
|
|
463
|
+
LEFT JOIN sources s ON s.id = sr.source_id
|
|
464
|
+
LEFT JOIN vector_index_entries v ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
|
|
465
|
+
WHERE v.id IS NULL
|
|
466
|
+
ORDER BY c.created_at ASC, c.ordinal ASC`).all(n.provider,n.model)}finally{i.close()}}function rr(e){x(e.dbPath);let t=b(e.dbPath);try{let r=t.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0,n=t.query("SELECT COUNT(*) AS n FROM chunks").get()?.n??0,i=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0,s=tr(e.dbPath,e).length,c=t.query(`SELECT COUNT(*) AS n FROM source_revisions
|
|
467
|
+
WHERE metadata_json LIKE '%"reindex_required":true%' OR metadata_json LIKE '%"status":"stale"%'`).get()?.n??0;return{schema_version:r,chunks:n,vector_entries:i,missing_embeddings:s,queued:Qi(e.dbPath),stale_revisions:c}}finally{t.close()}}function Ze(e){x(e.dbPath);let t=(e.now??new Date).toISOString(),r=e.reason??"missing_embedding",n=tr(e.dbPath,e),i=b(e.dbPath),s=0,c=0;try{i.transaction(()=>{for(let _ of n){let a=Vi("rq",`embedding\x00${_.chunk_id}\x00${r}`);if(i.query("SELECT id FROM reindex_queue WHERE kind = ? AND target_id = ? AND reason = ?").get("embedding",_.chunk_id,r)){c+=1;continue}i.run(`INSERT INTO reindex_queue (id, kind, target_id, source_uri, reason, status, metadata_json, created_at, updated_at)
|
|
468
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[a,"embedding",_.chunk_id,_.source_uri,r,"pending",JSON.stringify({source_revision_id:_.source_revision_id}),t,t]),s+=1}})()}finally{i.close()}return{enqueued:s,already_queued:c,reason:r}}function Zi(e){let t=b(e);try{let r=t.query("SELECT COUNT(*) AS n FROM chunk_embeddings").get()?.n??0,n=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0;return t.run("DELETE FROM vector_index_entries"),t.run("DELETE FROM chunk_embeddings"),{embeddings:r,vectorEntries:n}}finally{t.close()}}function es(e,t,r){let n=z(t.modelRef,t.config),i=L(n),s=b(e);try{return s.run(`UPDATE reindex_queue
|
|
469
|
+
SET status = ?, updated_at = ?
|
|
470
|
+
WHERE kind = ?
|
|
471
|
+
AND status = ?
|
|
472
|
+
AND EXISTS (
|
|
473
|
+
SELECT 1 FROM vector_index_entries v
|
|
474
|
+
WHERE v.chunk_id = reindex_queue.target_id
|
|
475
|
+
AND v.provider = ?
|
|
476
|
+
AND v.model = ?
|
|
477
|
+
)`,["completed",r,"embedding","pending",i.provider,i.model]).changes}finally{s.close()}}async function nr(e){x(e.dbPath);let t=(e.now??new Date).toISOString(),r=`run_${er()}`,n=e.full?Zi(e.dbPath):{embeddings:0,vectorEntries:0},i=Ze({...e,reason:e.full?"full_embedding_rebuild":"missing_embedding"}),s=b(e.dbPath);try{s.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
478
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[r,"embedding-refresh",e.full?"full":"incremental","running","local",z(e.modelRef,e.config),JSON.stringify({full:e.full===!0,queued:i}),t,t])}finally{s.close()}let c=await me({dbPath:e.dbPath,config:e.config,env:e.env,modelRef:e.modelRef,dimensions:e.dimensions,fake:e.fake,limit:e.limit,now:e.now}),u=es(e.dbPath,e,t),_=b(e.dbPath);try{_.run("UPDATE runs SET status = ?, metadata_json = ?, updated_at = ? WHERE id = ?",["completed",JSON.stringify({full:e.full===!0,queued:i,indexed:c,completed_queue_items:u}),t,r]),_.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
479
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${er()}`,r,"info","embedding_refresh_completed",JSON.stringify({queued:i,indexed:c,completed_queue_items:u}),t])}finally{_.close()}return{run_id:r,full:e.full===!0,deleted_embeddings:n.embeddings,deleted_vector_entries:n.vectorEntries,queued:i,indexed:c,completed_queue_items:u}}import{createHash as ts,randomUUID as ir}from"crypto";function rs(e){return`sha256:${ts("sha256").update(e).digest("hex")}`}function sr(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function or(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:{}}function H(e){return typeof e==="string"&&e.length>0?e:null}function ns(e){let t=or(e),r=H(t.url)??H(t.uri)??H(t.sourceUrl);if(!r)return null;return{url:r,title:H(t.title)??H(t.name),snippet:H(t.snippet)??H(t.text)??H(t.description),provider_metadata:t}}function Ie(e,t){if(Array.isArray(e)){for(let i of e)Ie(i,t);return}let r=ns(e);if(r)t.set(r.url,r);let n=or(e);for(let i of["sources","results","citations","annotations","output"])if(n[i])Ie(n[i],t)}function is(e,t){return Array.from({length:Math.min(t,3)},(r,n)=>({url:`https://example.com/knowledge-web-${n+1}`,title:`Fake web source ${n+1}`,snippet:`Deterministic web-search fixture for "${e}"`,provider_metadata:{fake:!0,rank:n+1}}))}async function ss(e){let{generateText:t}=await import("ai"),{createOpenAI:r}=await import("@ai-sdk/openai"),n=W(e.config,"openai"),i=r({apiKey:e.env[n.api_key_env],baseURL:n.base_url}),s=i.tools?.webSearch;if(!s)throw Error("OpenAI provider does not expose tools.webSearch.");return t({model:i(e.model),prompt:e.query,tools:{web_search:s({externalWebAccess:!0,searchContextSize:"medium",...e.domains.length>0?{allowedDomains:e.domains}:{}})},toolChoice:{type:"tool",toolName:"web_search"}})}async function os(e){let{generateText:t}=await import("ai"),{createAnthropic:r}=await import("@ai-sdk/anthropic"),n=W(e.config,"anthropic"),i=r({apiKey:e.env[n.api_key_env],baseURL:n.base_url}),s=i.tools?.webSearch_20250305??i.tools?.webSearch;if(!s)throw Error("Anthropic provider does not expose a web search tool.");return t({model:i(e.model),prompt:e.query,tools:{web_search:s({maxUses:e.maxUses,...e.domains.length>0?{allowedDomains:e.domains}:{}})}})}async function as(e,t,r){if(!e.fileResults||t.length===0)return 0;let n=t.map((s)=>{let c=[s.title,s.snippet,s.url].filter(Boolean).join(`
|
|
480
|
+
`),u=rs(c);return{source_ref:s.url,name:s.title??s.url,url:s.url,mime:"text/plain",hash:u,revision:u,status:"active",updated_at:r,permissions:{mode:"read_only",allowed_purposes:["knowledge_answer","knowledge_index"]},metadata:{source_ref:s.url,content_source:"provider_web_search",provider_metadata:s.provider_metadata},extracted_text:c}});return(await oe({dbPath:e.dbPath,items:n,sourceLabel:`web-search:${e.query}`,readAction:"provider_web_search_file_results",safetyPolicy:e.safetyPolicy,now:new Date(r)})).sources_upserted}async function ar(e){let t=e.query.trim();if(!t)throw Error("Web search query is required.");let r=e.env??process.env,n=(e.now??new Date).toISOString(),i=Math.max(1,Math.min(e.limit??5,20)),s=Math.max(1,Math.min(e.maxUses??3,10)),c=e.domains??[],u=X(e.modelRef??(e.provider?`${e.provider}:${W(e.config,e.provider).default_model}`:"default"),e.config),_=L(u),a=e.provider??_.provider,d=_.provider===a?_.model:W(e.config,a).default_model,o=`run_${ir()}`;if(!e.fake&&e.safetyPolicy)Q(e.safetyPolicy);if(!e.fake&&a!=="openai"&&a!=="anthropic")throw Error(`Provider ${a} does not expose native web search yet.`);if(!e.fake)B(a,e.config,r);x(e.dbPath);let l=b(e.dbPath);try{l.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
481
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[o,"provider-web-search",t,"running",a,d,JSON.stringify({domains:c,max_uses:s,fake:e.fake===!0}),n,n]),A(l,{event_type:"source_read",action:e.fake?"fake_provider_web_search":"provider_web_search",target_uri:t,decision:"allow",metadata:{provider:a,model:d,domains:c,max_uses:s},created_at:n})}finally{l.close()}let f="",g=[],E={input_tokens:sr(t),output_tokens:0,cost_usd:0},h=[];if(e.fake)g=is(t,i),f=`Fake web search answer for: ${t}`,E.output_tokens=sr(f);else{let k=a==="openai"?await ss({query:t,model:d,config:e.config,env:r,maxUses:s,domains:c}):await os({query:t,model:d,config:e.config,env:r,maxUses:s,domains:c});f=k.text;let p=new Map;Ie(k.sources,p),Ie(k.toolResults,p),g=Array.from(p.values()).slice(0,i);let R=fe({provider:a,model:d,usage:k.usage,providerMetadata:k.providerMetadata});E={input_tokens:R.input_tokens,output_tokens:R.output_tokens,cost_usd:R.cost_usd}}let m=await as(e,g,n),T=b(e.dbPath);try{T.run("UPDATE runs SET status = ?, metadata_json = ?, updated_at = ? WHERE id = ?",["completed",JSON.stringify({domains:c,max_uses:s,sources:g.length,filed_sources:m,fake:e.fake===!0}),n,o]),T.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
482
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${ir()}`,o,"info","provider_web_search_completed",JSON.stringify({sources:g.length,filed_sources:m}),n]),ge(T,{run_id:o,provider:a,model:d,input_tokens:E.input_tokens,output_tokens:E.output_tokens,cost_usd:E.cost_usd,metadata:{web_search:!0,sources:g.length,filed_sources:m},created_at:n})}finally{T.close()}if(g.length===0)h.push("no_web_sources_returned");return{run_id:o,query:t,provider:a,model:d,answer:f,sources:g,filed_sources:m,usage:E,warnings:h}}import{createHash as cs,randomUUID as ds}from"crypto";var cr=[{kind:"schema",prefix:"schemas/",description:"Machine-readable agent schemas and source rules."},{kind:"index",prefix:"indexes/",description:"Small orientation indexes and future shard manifests."},{kind:"log",prefix:"logs/",description:"Append-only JSONL run and wiki-maintenance log partitions."},{kind:"run",prefix:"runs/",description:"Prompt/tool/cost ledgers and generated output records."},{kind:"wiki_page",prefix:"wiki/",description:"Generated cited Markdown pages, not raw source files."},{kind:"export",prefix:"exports/",description:"Portable exports and snapshots of derived knowledge state."}];function dr(e){let t=typeof e==="string"?Buffer.from(e):Buffer.from(e);return{hash:`sha256:${cs("sha256").update(t).digest("hex")}`,size_bytes:t.byteLength}}function ur(e){return cr.find((r)=>e.startsWith(r.prefix))?.kind??"artifact"}function lr(e,t,r="global"){let n=et(e,t),i=e.storage.s3??null,s=i?.prefix?.replace(/^\/+|\/+$/g,"")??"",c=i?`s3://${i.bucket}/${s?`${s}/`:""}`:"";return{scope:r,mode:e.mode,storage_type:e.storage.type,workspace_home:t.home,local_layout:{app_path:ne,config_path:t.configPath,json_store_path:t.jsonStorePath,knowledge_db_path:t.knowledgeDbPath,directories:{artifacts:t.artifactsDir,cache:t.cacheDir,exports:t.exportsDir,indexes:t.indexesDir,logs:t.logsDir,runs:t.runsDir,schemas:t.schemasDir,wiki:t.wikiDir}},artifact_store:{type:e.storage.type,artifacts_root:e.storage.artifacts_root,uri_prefix:e.storage.type==="s3"?c:`file://${t.artifactsDir}/`,s3:i?{bucket:i.bucket,prefix:s,region:i.region??null,profile:i.profile??null,server_side_encryption:i.server_side_encryption??null,kms_key_configured:Boolean(i.kms_key_id)}:null},source_ownership:{owner:"open-files",preferred_ref:e.sources.preferred_ref,allowed_schemes:e.sources.allowed_schemes,raw_source_bytes_stored_in_open_knowledge:!1,stores:["source refs","source revisions and hashes","citation spans","redacted extracted chunks","embeddings","generated wiki artifacts","indexes","run ledgers"],does_not_store:["raw open-files bytes","S3 object credentials","connector secrets","hosted tenant ownership state"]},generated_artifacts:cr,scalability:{catalog:"knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.",indexes:"Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.",logs:"Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.",markdown:"Markdown pages are the readable wiki layer over DB/object-store state."},warnings:n.warnings}}function et(e,t){let r=[],n=[];if(!t.home.endsWith(ne))n.push(`Workspace home does not end with ${ne}: ${t.home}`);if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)r.push("storage.s3.bucket is required when storage.type is s3.");if(!e.storage.s3?.prefix)n.push("storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.");if(e.mode==="local")n.push("storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.")}if(e.storage.type==="local"&&e.storage.s3)n.push("storage.s3 is configured but ignored while storage.type is local.");if(e.sources.preferred_ref!=="open-files")n.push("sources.preferred_ref should stay open-files for durable company knowledge.");if(!e.sources.allowed_schemes.includes("open-files"))r.push("sources.allowed_schemes must include open-files.");return{ok:r.length===0,errors:r,warnings:n}}function _r(e,t,r=new Date){let n=r.toISOString(),i=e.prepare(`
|
|
443
483
|
INSERT INTO storage_objects (
|
|
444
484
|
id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
|
|
445
485
|
)
|
|
@@ -451,7 +491,7 @@ ${l}`;else{let{generateText:p}=await import("ai"),x=await kt(i,{config:e.config,
|
|
|
451
491
|
size_bytes = excluded.size_bytes,
|
|
452
492
|
metadata_json = excluded.metadata_json,
|
|
453
493
|
updated_at = excluded.updated_at
|
|
454
|
-
`);e.transaction((c)=>{for(let u of c)i.run(
|
|
494
|
+
`);e.transaction((c)=>{for(let u of c)i.run(ds(),u.uri,u.kind,u.content_type??null,u.hash??null,u.size_bytes??null,JSON.stringify({key:u.key,...u.metadata??{}}),n,n)})(t)}import{createHash as us}from"crypto";function ls(e){let t=String(e.getUTCFullYear()),r=String(e.getUTCMonth()+1).padStart(2,"0"),n=String(e.getUTCDate()).padStart(2,"0");return{year:t,month:r,day:n}}function tt(e,t){return`${e}_${us("sha256").update(t).digest("hex").slice(0,20)}`}function _s(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function fs(){return`# Knowledge Agent Schema v1
|
|
455
495
|
|
|
456
496
|
## Source Rules
|
|
457
497
|
|
|
@@ -476,7 +516,7 @@ ${l}`;else{let{generateText:p}=await import("ai"),x=await kt(i,{config:e.config,
|
|
|
476
516
|
## Lint Rules
|
|
477
517
|
|
|
478
518
|
- Flag stale pages, missing citations, contradictions, orphan pages, duplicate pages, and unresolved source refs.
|
|
479
|
-
`}function
|
|
519
|
+
`}function gs(){return`# Knowledge Index
|
|
480
520
|
|
|
481
521
|
This is a compact orientation index for agents. It is not the full search index.
|
|
482
522
|
|
|
@@ -491,19 +531,19 @@ This is a compact orientation index for agents. It is not the full search index.
|
|
|
491
531
|
|
|
492
532
|
Raw source files are resolved through open-files. This app stores source refs,
|
|
493
533
|
citations, chunks, generated wiki artifacts, indexes, and run records.
|
|
494
|
-
`}function
|
|
534
|
+
`}function fr(){return`# Wiki
|
|
495
535
|
|
|
496
536
|
Generated durable knowledge pages live here.
|
|
497
537
|
|
|
498
538
|
Pages should be concise, cited, and organized for both humans and agents.
|
|
499
|
-
`}async function
|
|
500
|
-
`,content_type:"application/x-ndjson"}],o=await Promise.all(d.map(async(l)=>{let f=await e.put(l);return{key:f.key,uri:f.uri,kind:
|
|
501
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[u,t,"wiki",0,i,
|
|
539
|
+
`}async function gr(e,t=new Date){let{year:r,month:n,day:i}=ls(t),s="schemas/v1.md",c="indexes/root.md",u="wiki/README.md",_=`logs/${r}/${n}/${i}.jsonl`,a={ts:t.toISOString(),event:"wiki_layout_initialized",schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md"},d=[{key:"schemas/v1.md",body:fs(),content_type:"text/markdown"},{key:"indexes/root.md",body:gs(),content_type:"text/markdown"},{key:"wiki/README.md",body:fr(),content_type:"text/markdown"},{key:_,body:`${JSON.stringify(a)}
|
|
540
|
+
`,content_type:"application/x-ndjson"}],o=await Promise.all(d.map(async(l)=>{let f=await e.put(l);return{key:f.key,uri:f.uri,kind:ur(l.key),content_type:l.content_type,metadata:{provenance:qe({generated_from:"wiki_layout_init",artifact_key:l.key,citation_required:l.key.startsWith("wiki/")||l.key.startsWith("indexes/")})},...dr(l.body)}}));return{schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md",log_key:_,artifacts:o,written:["schemas/v1.md","indexes/root.md","wiki/README.md",_]}}function rt(e){let t=e.metadata?.provenance;if(t&&typeof t==="object"&&!Array.isArray(t))return t;return qe({generated_from:"wiki_layout_init",artifact_key:e.key})}function ps(e,t,r,n,i,s){let c=rt(n),u=tt("chk",`${t}\x00${n.hash??n.uri}`),_=e.query("SELECT id FROM chunks WHERE wiki_page_id = ?").all(t);for(let a of _)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[a.id]);e.run("DELETE FROM chunks WHERE wiki_page_id = ?",[t]),e.run(`INSERT INTO chunks (id, wiki_page_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
541
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[u,t,"wiki",0,i,_s(i),0,i.length,JSON.stringify({artifact_key:n.key,artifact_uri:n.uri,content_hash:n.hash??null,provenance:c}),s]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[u,i,r,n.uri])}function pr(e,t,r=new Date){let n=r.toISOString(),i=t.find((c)=>c.key.endsWith("indexes/root.md")),s=t.find((c)=>c.key.endsWith("wiki/README.md"));if(i)e.run(`INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
|
|
502
542
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
503
543
|
ON CONFLICT(kind, name, shard_key) DO UPDATE SET
|
|
504
544
|
artifact_uri = excluded.artifact_uri,
|
|
505
545
|
metadata_json = excluded.metadata_json,
|
|
506
|
-
updated_at = excluded.updated_at`,[
|
|
546
|
+
updated_at = excluded.updated_at`,[tt("idx","root:indexes/root.md"),"root","root",i.uri,"root",JSON.stringify({artifact_key:i.key,content_hash:i.hash??null,provenance:rt(i)}),n,n]);if(s){let c=tt("wiki","wiki/README.md");e.run(`INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
|
|
507
547
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
508
548
|
ON CONFLICT(path) DO UPDATE SET
|
|
509
549
|
title = excluded.title,
|
|
@@ -511,7 +551,7 @@ Pages should be concise, cited, and organized for both humans and agents.
|
|
|
511
551
|
content_hash = excluded.content_hash,
|
|
512
552
|
status = excluded.status,
|
|
513
553
|
metadata_json = excluded.metadata_json,
|
|
514
|
-
updated_at = excluded.updated_at`,[c,"wiki/README.md","Wiki",s.uri,s.hash??null,"active",JSON.stringify({artifact_key:s.key,provenance:
|
|
554
|
+
updated_at = excluded.updated_at`,[c,"wiki/README.md","Wiki",s.uri,s.hash??null,"active",JSON.stringify({artifact_key:s.key,provenance:rt(s)}),n,n]),ps(e,c,"Wiki",s,fr(),n)}}class mr{options;ensuredWorkspace;cachedConfig;constructor(e={}){this.options=e}get scope(){return this.options.scope??"global"}get workspace(){return this.ensuredWorkspace??st(this.options.scope,this.options.cwd)}ensureWorkspace(){if(!this.ensuredWorkspace)this.ensuredWorkspace=it(this.workspace.home);return this.ensuredWorkspace}jsonStorePath(){return this.ensureWorkspace().jsonStorePath}config(){if(!this.cachedConfig){let e=this.ensureWorkspace();this.cachedConfig=ot(e.configPath)}return this.cachedConfig}safetyPolicy(){return Ht(this.config(),this.ensureWorkspace())}artifactStore(){return gt(this.config(),this.ensureWorkspace())}storageContract(){return lr(this.config(),this.ensureWorkspace(),this.scope)}validateStorage(){return et(this.config(),this.ensureWorkspace())}paths(){let e=this.ensureWorkspace();return{ok:!0,scope:this.scope,home:e.home,config_path:e.configPath,json_store_path:e.jsonStorePath,knowledge_db_path:e.knowledgeDbPath,artifacts_dir:e.artifactsDir,indexes_dir:e.indexesDir,logs_dir:e.logsDir,runs_dir:e.runsDir,schemas_dir:e.schemasDir,wiki_dir:e.wikiDir,config:this.config(),message:e.home}}initDb(){return x(this.ensureWorkspace().knowledgeDbPath)}dbStats(){let e=this.ensureWorkspace();return x(e.knowledgeDbPath),ut(e.knowledgeDbPath)}async initWiki(){let e=this.ensureWorkspace();x(e.knowledgeDbPath);let t=await gr(this.artifactStore()),r=b(e.knowledgeDbPath);try{_r(r,t.artifacts),pr(r,t.artifacts)}finally{r.close()}return t}async ingestManifest(e){let t=this.ensureWorkspace();return Yt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}async ingestSource(e,t){let r=this.ensureWorkspace();return Zt({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t,config:this.config(),safetyPolicy:this.safetyPolicy()})}async resolveSource(e,t={}){let r=this.ensureWorkspace();return Oe({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t.purpose,limit:t.limit,safetyPolicy:this.safetyPolicy()})}async consumeOutbox(e){let t=this.ensureWorkspace();return Gt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}reindexHealth(e={}){let t=this.ensureWorkspace();return rr({...e,dbPath:t.knowledgeDbPath,config:this.config()})}enqueueReindex(e={}){let t=this.ensureWorkspace();return Ze({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async refreshEmbeddings(e={}){let t=this.ensureWorkspace();return nr({...e,dbPath:t.knowledgeDbPath,config:this.config()})}providerStatus(e=process.env){return yt(this.config(),e)}modelRegistry(){return Xe(this.config())}embeddingStatus(){let e=this.ensureWorkspace();return Rt(e.knowledgeDbPath)}async indexEmbeddings(e={}){let t=this.ensureWorkspace();return me({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async semanticSearch(e){let t=this.ensureWorkspace();return he({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async search(e){let t=this.ensureWorkspace();return be({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async retrieveContext(e){let t=this.ensureWorkspace();return we({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async runPrompt(e){let t=this.ensureWorkspace();return Kt({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async webSearch(e){let t=this.ensureWorkspace();return ar({...e,dbPath:t.knowledgeDbPath,config:this.config(),safetyPolicy:this.safetyPolicy()})}}function hr(e={}){return new mr(e)}import{basename as hs}from"path";var ae={name:"@hasna/knowledge",version:"0.2.19",description:"Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",type:"module",bin:{knowledge:"bin/open-knowledge.js","open-knowledge":"bin/open-knowledge.js","open-knowledge-mcp":"bin/open-knowledge-mcp.js"},files:["bin","src","docs","LICENSE","README.md"],scripts:{test:"bun test","test:cli":"bun test tests/cli.test.ts",build:"bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",prepublishOnly:"bun run build",postinstall:"bun run build"},keywords:["knowledge","cli","agents","json","notes","local","store"],license:"Apache-2.0",publishConfig:{registry:"https://registry.npmjs.org",access:"public"},repository:{type:"git",url:"git+https://github.com/hasna/knowledge.git"},bugs:{url:"https://github.com/hasna/knowledge/issues"},author:"Hasna Inc. <hasna@example.com>",engines:{bun:">=1.0",node:">=18"},dependencies:{"@aws-sdk/client-s3":"^3.1063.0","@aws-sdk/credential-providers":"^3.1063.0","@ai-sdk/anthropic":"^3.0.81","@ai-sdk/deepseek":"^2.0.35","@ai-sdk/openai":"^3.0.68","@modelcontextprotocol/sdk":"^1.29.0",ai:"^6.0.197",zod:"^4.3.6"},devDependencies:{"@types/bun":"^1.3.14"}};var Er={debug:0,info:1,warn:2,error:3},Es=()=>{if(process.env.DEBUG)return"debug";if(process.env.LOG_LEVEL==="debug")return"debug";if(process.env.LOG_LEVEL==="warn")return"warn";if(process.env.LOG_LEVEL==="error")return"error";return"info"};function Y(e,t,r){if(Er[e]<Er[Es()])return;let n={debug:"[DEBUG]",info:"[INFO]",warn:"[WARN]",error:"[ERROR]"}[e],i=r?`${n} ${t} ${JSON.stringify(r)}`:`${n} ${t}`;if(e==="error")console.error(i);else console.error(i)}var yr=["add","list","get","delete","update","archive","restore","upsert","untag","export","prune","dedupe","stats","paths","storage","db","wiki","source","ingest","reindex","search","web","ask","build","embeddings","providers","safety","help"],kr={ls:"list",rm:"delete",edit:"update",unarchive:"restore",knowledge:"ask"};function ys(e){let t=[],r={};for(let n=0;n<e.length;n+=1){let i=e[n];if(!i.startsWith("-")){t.push(i);continue}switch(i){case"--json":r.json=!0;break;case"--yes":case"-y":r.yes=!0;break;case"--help":case"-h":r.help=!0;break;case"--version":case"-v":r.version=!0;break;case"--desc":r.desc=!0;break;case"--page":case"-p":r.page=Number(e[n+1]),n+=1;break;case"--limit":case"-l":r.limit=Number(e[n+1]),n+=1;break;case"--search":case"-s":r.search=e[n+1],n+=1;break;case"--sort":r.sort=e[n+1],n+=1;break;case"--id":r.id=e[n+1],n+=1;break;case"--store":r.store=e[n+1],n+=1;break;case"--title":r.title=e[n+1],n+=1;break;case"--content":r.content=e[n+1],n+=1;break;case"--url":r.url=e[n+1],n+=1;break;case"--tag":case"-t":r.tag=e[n+1],n+=1;break;case"--format":r.format=e[n+1],n+=1;break;case"--completions":r.completions=e[n+1],n+=1;break;case"--purpose":r.purpose=e[n+1],n+=1;break;case"--model":r.model=e[n+1],n+=1;break;case"--dimensions":r.dimensions=Number(e[n+1]),n+=1;break;case"--semantic":r.semantic=!0;break;case"--context":r.context=!0;break;case"--generate":r.generate=!0;break;case"--approve-write":r.approveWrite=!0;break;case"--provider":r.provider=e[n+1],n+=1;break;case"--domain":r.domain=[...r.domain??[],e[n+1]],n+=1;break;case"--file-results":r.fileResults=!0;break;case"--full":r.full=!0;break;case"--fake":r.fake=!0;break;case"--no-color":r.noColor=!0;break;case"--scope":r.scope=e[n+1],n+=1;break;case"--older-than":r.olderThan=Number(e[n+1]),n+=1;break;case"--empty":r.empty=!0;break;case"--archived":r.archived=!0;break;case"--include-archived":r.includeArchived=!0;break;default:throw Error(`Unknown flag: ${i}. Run 'open-knowledge --help' for valid options.`)}}return{positional:t,flags:r}}function ks(e){if(!e)return"";return kr[e]??e}function bs(e,t){let r=Array.from({length:e.length+1},()=>Array(t.length+1).fill(0));for(let n=0;n<=e.length;n+=1)r[n][0]=n;for(let n=0;n<=t.length;n+=1)r[0][n]=n;for(let n=1;n<=e.length;n+=1)for(let i=1;i<=t.length;i+=1){let s=e[n-1]===t[i-1]?0:1;r[n][i]=Math.min(r[n-1][i]+1,r[n][i-1]+1,r[n-1][i-1]+s)}return r[e.length][t.length]}function vs(e){if(!e)return"";let t=[...yr,...Object.keys(kr)],r="",n=Number.POSITIVE_INFINITY;for(let i of t){let s=bs(e,i);if(s<n)n=s,r=i}return n<=3?r:""}function ws(){return hs(process.argv[1]??"")==="knowledge"}function Ts(){console.log(`open-knowledge - local agent knowledge store
|
|
515
555
|
|
|
516
556
|
Usage:
|
|
517
557
|
open-knowledge <command> [options]
|
|
@@ -537,7 +577,7 @@ Commands:
|
|
|
537
577
|
source resolve <source-ref> Resolve read-only source content and citation evidence
|
|
538
578
|
ingest manifest <file|s3://> Ingest an open-files manifest into knowledge.db
|
|
539
579
|
ingest source <source-ref> Ingest a read-only source ref into knowledge.db
|
|
540
|
-
reindex outbox
|
|
580
|
+
reindex status|enqueue|embeddings|outbox Inspect/refresh search indexes
|
|
541
581
|
search <query> Hybrid search sources, wiki pages, indexes, or context
|
|
542
582
|
web search <query> Provider-native web search with citations
|
|
543
583
|
ask|build <prompt> Build a read-only citation answer/context pack
|
|
@@ -559,6 +599,7 @@ Global Options:
|
|
|
559
599
|
--provider <name> Provider override for web search
|
|
560
600
|
--domain <domain> Restrict provider web search to a domain
|
|
561
601
|
--file-results File web snippets as web source refs
|
|
602
|
+
--full Force full embedding index rebuild
|
|
562
603
|
--fake Use deterministic fake embeddings for local tests
|
|
563
604
|
--scope local|global|project Store scope (default: global ~/.hasna/apps/knowledge/)
|
|
564
605
|
--no-color Disable color output
|
|
@@ -596,5 +637,5 @@ Export Options:
|
|
|
596
637
|
|
|
597
638
|
Prune Options:
|
|
598
639
|
--older-than <days> Remove items older than N days
|
|
599
|
-
--empty Remove items with empty content`)}function
|
|
600
|
-
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--file-results)--file-results" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--provider)--provider[provider]:" "(--domain)--domain[domain]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l provider; complete -c open-knowledge -l domain; complete -c open-knowledge -l file-results; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=ds(t[0]),i=1;if(_s()&&n&&!gr.includes(n))n="ask",i=0;if(!n||r.help||n==="help"){gs(t[1]);return}let s=_r({scope:r.scope}),c=r.store;if(!c)if(r.scope==="project"||r.scope==="local")c=s.jsonStorePath();else c=Pe();if(n==="paths"){E(s.paths(),r.json);return}if(n==="storage"){let a=t[1]??"status";if(a==="status"){let d=s.storageContract(),o=s.validateStorage();E({ok:o.ok,...d,validation:o,message:`${d.storage_type} artifact storage at ${d.artifact_store.uri_prefix}`},r.json);return}if(a==="validate"){let d=s.validateStorage();E({ok:d.ok,validation:d,message:d.ok?"Storage contract valid":`Storage contract invalid: ${d.errors.join("; ")}`},r.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(n==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let o=s.initDb();E({ok:!0,...o,message:`Initialized ${o.path}`},r.json);return}let d=s.dbStats();E({ok:!0,path:s.workspace.knowledgeDbPath,...d,message:`knowledge.db schema v${d.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let d=await s.initWiki();E({ok:!0,...d,message:`Initialized wiki layout in ${s.workspace.home}`},r.json);return}if(n==="safety"){let a=t[1]??"status",d=s.ensureWorkspace(),o=s.safetyPolicy();s.initDb();let l=w(d.knowledgeDbPath);try{if(a==="status"){E({ok:!0,mode:o.mode,workspace:d.home,allow_write_roots:o.allowWriteRoots,read_only_source_access:o.readOnlySourceAccess,network:o.network,redaction:o.redaction,approvals:o.approvals,message:`Safety policy: ${o.mode}`},r.json);return}if(a==="check"){let f=t[2]??"generated_write",g=t[3]??null,k;try{if(f==="web_search")Y(o),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!g)throw Error("safety check s3_read requires an s3:// target.");q(g,o),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"}}else k=Bt(l,o,f,g);A(l,{event_type:"safety_check",action:f,target_uri:g,decision:k.decision==="allow"?"allow":"requires_approval",metadata:k}),E({ok:!0,...k,message:`Safety check ${k.decision}`},r.json);return}catch(h){throw A(l,{event_type:"safety_check",action:f,target_uri:g,decision:"deny",metadata:{error:h instanceof Error?h.message:String(h)}}),h}}if(a==="approve"){let f=t[2]??"generated_write",g=t[3]??null,k=Ht(l,{action:f,target_uri:g,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});A(l,{event_type:"approval",action:f,target_uri:g,decision:"allow",metadata:{approval_id:k.id}}),E({ok:!0,...k,action:f,target_uri:g,message:`Approved ${f}`},r.json);return}if(a==="audit"){let f=l.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((g)=>({id:g.id,event_type:g.event_type,action:g.action,target_uri:g.target_uri,decision:g.decision,metadata:JSON.parse(g.metadata_json),created_at:g.created_at}));E({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(a==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let g=ve(f,o);if(g.findings.length>0)we(l,{source_uri:"safety://redact",findings:g.findings,metadata:{command:"safety redact"}});A(l,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:g.findings.length>0?"redacted":"allow",metadata:{findings:g.findings.length}}),E({ok:!0,text:g.text,findings:g.findings,message:`Redacted ${g.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{l.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge source resolve <source-ref>");let o=await s.resolveSource(d,{purpose:r.purpose,limit:r.limit});E({ok:!0,...o,message:o.resolved?`Resolved ${o.source_ref} (${o.content.chunks_returned}/${o.content.chunks_total} chunks)`:`Source not indexed: ${d}`},r.json);return}if(n==="ingest"){let a=t[1]??"";if(a==="manifest"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let o=await s.ingestManifest(d);E({ok:!0,...o,message:`Ingested ${o.items_seen} manifest item(s)`},r.json);return}if(a==="source"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest source <source-ref>");let o=await s.ingestSource(d,r.purpose);E({ok:!0,...o,message:`Ingested source ${o.source_ref} (${o.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let o=await s.consumeOutbox(d);E({ok:!0,...o,message:`Consumed ${o.events_seen} outbox event(s)`},r.json);return}if(n==="embeddings"){let a=t[1]??"status";if(a==="status"){let d=s.embeddingStatus();E({ok:!0,...d,message:`${d.total_vector_entries} vector index entries`},r.json);return}if(a==="index"){let d=await s.indexEmbeddings({limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`Embedded ${d.chunks_embedded} chunk(s)`},r.json);return}if(a==="search"){let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge embeddings search <query>");let o=await s.semanticSearch({query:d,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...o,message:`${o.results.length} semantic result(s)`},r.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(n==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");if(r.context){let o=await s.retrieveContext({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...o,message:`${o.excerpts.length} context excerpt(s)`},r.json);return}let d=await s.search({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`${d.results.length} search result(s)`},r.json);return}if(n==="web"){if((t[1]??"search")!=="search")throw Error("Invalid web action. Use 'search'.");let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge web search <query>");let o=await s.webSearch({query:d,limit:r.limit,modelRef:r.model,provider:r.provider,domains:r.domain,fake:r.fake,fileResults:r.fileResults});E({ok:!0,...o,message:`${o.sources.length} web source(s)`},r.json);return}if(n==="ask"||n==="build"){let a=t.slice(i).join(" ");if(!a)throw Error("Usage: open-knowledge ask <prompt>");let d=await s.runPrompt({prompt:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake,generate:r.generate,approveWrite:r.approveWrite});E({ok:!0,...d,message:d.generated?"Generated answer with citations":"Prepared citation context draft"},r.json);return}if(n==="providers"){let a=t[1]??"status";if(a==="status"){let d=s.providerStatus(),o=d.providers.filter((l)=>l.configured).length;E({ok:!0,...d,message:`${o}/${d.providers.length} provider credential(s) configured`},r.json);return}if(a==="models"){let d=s.modelRegistry();E({ok:!0,models:d,message:`${d.length} model alias(es)`},r.json);return}if(a==="check"){let d=t[2]??"default",o=$(d,s.config()),l=U(o),f=B(l.provider,s.config());E({ok:!0,target:d,model_ref:o,provider:l.provider,model:l.model,credential:f,message:`${l.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(De(c),n==="add"){let a=t[1],d=t[2];if(!a||!d)throw Error("Usage: open-knowledge add <title> <content>");P(c,()=>{let o=C(c),l={id:Ue(),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};o.items.push(l),M(c,o),J("info","Item added",{id:l.id,title:l.title}),E({ok:!0,item:l,message:`Added ${l.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");P(c,()=>{let a=C(c),d=Number.isFinite(r.page)&&r.page>0?r.page:1,o=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,l=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",g=r.format==="table"||!r.json&&!r.format&&ps(r),k=r.json||r.format==="json",h=a.items;if(r.archived)h=h.filter((b)=>b.archived===!0);else if(!r.includeArchived)h=h.filter((b)=>!b.archived);if(l)h=h.filter((b)=>b.title.toLowerCase().includes(l)||b.content.toLowerCase().includes(l));if(f)h=h.filter((b)=>b.tags&&b.tags.map((Ae)=>Ae.toLowerCase()).includes(f));let{sorted:m,sort:S,direction:y}=ms(h,r),p=(d-1)*o,x=m.slice(p,p+o),N=Math.max(1,Math.ceil(m.length/o));if(k){E({ok:!0,page:d,limit:o,total:m.length,total_pages:N,sort:S,direction:y,items:x},!0);return}if(x.length===0){E(`No items found (search=${l||"none"}, tag=${f||"none"})`,!1);return}if(g){let b=(F)=>F,Ae=`${b("ID")} ${b("TITLE")} ${b("CREATED")} ${b("URL")} ${b("TAGS")}`;console.log(Ae);for(let F of x)console.log(`${F.id} ${b(F.title)} ${F.created_at} ${F.url?b(F.url):""} ${F.tags?.length?b(`[${F.tags.join(", ")}]`):""}`);console.log(`Page ${d}/${N} | showing ${x.length} of ${m.length} | sort=${S} ${y} | search=${l||"none"} | tag=${f||"none"}`)}else{for(let b of x)console.log(`${b.id} ${b.title} ${b.created_at}${b.url?` ${b.url}`:""}${b.tags?.length?` [${b.tags.join(", ")}]`:""}`);console.log(`Page ${d}/${N} | showing ${x.length} of ${m.length} | sort=${S} ${y} | search=${l||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){ae(r),P(c,()=>{let d=C(c).items.find((o)=>o.id===r.id||o.short_id===r.id);if(!d)throw Error(`Item not found: ${r.id}`);E({ok:!0,item:d,message:`${d.id}: ${d.title}`},r.json)});return}if(n==="update"){ae(r),P(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];if(r.title!==void 0)o.title=r.title;if(r.content!==void 0)o.content=r.content;if(r.url!==void 0)o.url=r.url;if(r.tag!==void 0){if(o.tags=o.tags||[],!o.tags.map((l)=>l.toLowerCase()).includes(r.tag.toLowerCase()))o.tags.push(r.tag)}o.updated_at=new Date().toISOString(),a.items[d]=o,M(c,a),E({ok:!0,item:o,message:`Updated ${o.id}`},r.json)});return}if(n==="archive"||n==="restore"){ae(r),P(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];o.archived=n==="archive",o.updated_at=new Date().toISOString(),a.items[d]=o,M(c,a),E({ok:!0,item:o,message:`${n==="archive"?"Archived":"Restored"} ${o.id}`},r.json)});return}if(n==="untag"){if(ae(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");P(c,()=>{let a=C(c),d=a.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d],l=o.tags?.length??0;o.tags=(o.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),o.updated_at=new Date().toISOString(),a.items[d]=o,M(c,a),E({ok:!0,item:o,removed:l-o.tags.length,message:`Removed tag from ${o.id}`},r.json)});return}if(n==="upsert"){let a=r.title??t[1],d=r.content??t[2];P(c,()=>{let o=C(c),l=r.id?o.items.findIndex((k)=>k.id===r.id||k.short_id===r.id):-1,f=new Date().toISOString();if(l===-1){if(!a||!d)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let k=r.id??Ue(),h={id:k,short_id:at(k),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};o.items.push(h),M(c,o),E({ok:!0,created:!0,item:h,message:`Upserted ${h.id}`},r.json);return}let g=o.items[l];if(a!==void 0)g.title=a;if(d!==void 0)g.content=d;if(r.url!==void 0)g.url=r.url;if(r.tag!==void 0){if(g.tags=g.tags||[],!g.tags.map((k)=>k.toLowerCase()).includes(r.tag.toLowerCase()))g.tags.push(r.tag)}g.updated_at=f,o.items[l]=g,M(c,o),E({ok:!0,created:!1,item:g,message:`Upserted ${g.id}`},r.json)});return}if(n==="delete"){if(ae(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");P(c,()=>{let a=C(c),d=a.items.length;a.items=a.items.filter((l)=>l.id!==r.id&&l.short_id!==r.id);let o=d!==a.items.length;if(M(c,a),!o)throw Error(`Item not found: ${r.id}`);J("info","Item deleted",{id:r.id}),E({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let a=r.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");P(c,()=>{let d=C(c);if(a==="jsonl")for(let o of d.items)console.log(JSON.stringify(o));else E({ok:!0,items:d.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");P(c,()=>{let a=C(c),d=a.items.length;if(r.olderThan!==void 0){let l=new Date;l.setDate(l.getDate()-r.olderThan),a.items=a.items.filter((f)=>new Date(f.created_at)>=l)}if(r.empty)a.items=a.items.filter((l)=>l.content.trim().length>0);let o=d-a.items.length;M(c,a),J("info","Prune completed",{pruned:o,remaining:a.items.length}),E({ok:!0,pruned:o,remaining:a.items.length,message:`Pruned ${o} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");P(c,()=>{let a=C(c),d=new Set,o=a.items.length;a.items=a.items.filter((f)=>{let g=`${f.title}\x00${f.content}`;if(d.has(g))return!1;return d.add(g),!0});let l=o-a.items.length;M(c,a),J("info","Dedupe completed",{removed:l,remaining:a.items.length}),E({ok:!0,removed:l,remaining:a.items.length,message:`Dedupe removed ${l} duplicate(s)`},r.json)});return}if(n==="stats"){P(c,()=>{let a=C(c),d=a.items.filter((y)=>!y.archived),o=d.length,l=a.items.length-o,f=d.filter((y)=>y.url).length,g=d.filter((y)=>y.tags&&y.tags.length>0).length,k=o>0?d.map((y)=>y.created_at).sort()[0]:null,h=o>0?d.map((y)=>y.created_at).sort()[o-1]:null,m={};for(let y of d)for(let p of y.tags||[])m[p]=(m[p]||0)+1;let S=Object.entries(m).sort((y,p)=>p[1]-y[1]).slice(0,5).map(([y,p])=>({tag:y,count:p}));E({ok:!0,total:o,archived:l,with_url:f,with_tags:g,oldest:k,newest:h,top_tags:S,message:`${o} items | ${f} with URL | ${g} with tags`},r.json)});return}let u=ls(t[0]),_=u?` Did you mean '${u}'?`:"";throw J("warn","Unknown command",{input:t[0],suggestion:u}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)hs(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);J("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{ls as suggestCommand,ms as sortItems,hs as run,cs as parseArgs};
|
|
640
|
+
--empty Remove items with empty content`)}function xs(e){if(e==="add"){console.log("Usage: open-knowledge add <title> <content> [--url <url>] [-t <tag>] [--json]");return}if(e==="list"||e==="ls"){console.log("Usage: open-knowledge list|ls [--format table|json] [-p <page>] [-l <limit>] [-s <search>] [-t <tag>] [--sort created|title] [--desc] [--json]");return}if(e==="get"){console.log("Usage: open-knowledge get --id <id> [--json]");return}if(e==="update"||e==="edit"){console.log("Usage: open-knowledge update|edit --id <id> [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="archive"){console.log("Usage: open-knowledge archive --id <id> [--json]");return}if(e==="restore"||e==="unarchive"){console.log("Usage: open-knowledge restore|unarchive --id <id> [--json]");return}if(e==="upsert"){console.log("Usage: open-knowledge upsert [title] [content] [--id <id>] [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="untag"){console.log("Usage: open-knowledge untag --id <id> -t <tag> [--json]");return}if(e==="delete"||e==="rm"){console.log("Usage: open-knowledge delete|rm --id <id> -y [--json]");return}if(e==="export"){console.log("Usage: open-knowledge export [--format jsonl] [--json]");return}if(e==="prune"){console.log("Usage: open-knowledge prune --yes [--older-than <days>] [--empty] [--json]");return}if(e==="dedupe"){console.log("Usage: open-knowledge dedupe --yes [--json]");return}if(e==="stats"){console.log("Usage: open-knowledge stats [--json]");return}if(e==="paths"){console.log("Usage: open-knowledge paths [--scope local|global|project] [--json]");return}if(e==="storage"){console.log("Usage: open-knowledge storage status|validate [--scope local|global|project] [--json]");return}if(e==="db"){console.log("Usage: open-knowledge db init|stats [--scope local|global|project] [--json]");return}if(e==="wiki"){console.log("Usage: open-knowledge wiki init [--scope local|global|project] [--json]");return}if(e==="source"){console.log("Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]");return}if(e==="ingest"){console.log("Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]");return}if(e==="reindex"){console.log("Usage: open-knowledge reindex status|enqueue|embeddings|outbox [file|s3://bucket/key] [--full] [--fake] [--scope local|global|project] [--json]");return}if(e==="search"){console.log("Usage: open-knowledge search <query> [--context] [--semantic] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="web"){console.log("Usage: open-knowledge web search <query> [--provider openai|anthropic] [--model provider:model] [--domain <domain>] [--file-results] [--fake] [--scope local|global|project] [--json]");return}if(e==="ask"||e==="build"||e==="knowledge"){console.log("Usage: open-knowledge ask|build <prompt> [--generate] [--semantic] [--model default|provider:model] [--approve-write] [--scope local|global|project] [--json]");return}if(e==="embeddings"){console.log("Usage: open-knowledge embeddings status|index|search [query] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="providers"){console.log("Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]");return}if(e==="safety"){console.log("Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]");return}Ts()}function Ss(e){if(e.noColor||process.env.NO_COLOR)return!1;if(process.env.FORCE_COLOR)return!0;return process.stdout.isTTY===!0}function y(e,t,r){if(t){console.log(JSON.stringify(e,null,2));return}if(typeof e==="string"){console.log(e);return}console.log(e.message??JSON.stringify(e,null,2))}function ce(e){if(!e.id)throw Error("Missing required --id. Example: open-knowledge get --id <id>")}function Rs(e,t){let r=t.sort??"created";if(r!=="created"&&r!=="title")throw Error("Invalid --sort value. Use 'created' or 'title'.");let n=[...e].sort((i,s)=>{if(r==="title")return i.title.localeCompare(s.title);return i.created_at.localeCompare(s.created_at)});if(t.desc)n.reverse();return{sorted:n,sort:r,direction:t.desc?"desc":"asc"}}async function Os(e){let{positional:t,flags:r}=ys(e);if(Y("debug","CLI invoked",{command:t[0],flags:{json:r.json,store:r.store}}),r.version){console.log(r.json?JSON.stringify({name:ae.name,version:ae.version},null,2):`${ae.name} ${ae.version}`);return}if(r.completions){let a=r.completions;if(a==="bash")console.log('_open_knowledge() { local cur; cur="${COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --semantic --context --generate --approve-write --provider --domain --file-results --full --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge');else if(a==="zsh")console.log(`#compdef open-knowledge
|
|
641
|
+
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--file-results)--file-results" "(--full)--full" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--provider)--provider[provider]:" "(--domain)--domain[domain]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l provider; complete -c open-knowledge -l domain; complete -c open-knowledge -l file-results; complete -c open-knowledge -l full; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=ks(t[0]),i=1;if(ws()&&n&&!yr.includes(n))n="ask",i=0;if(!n||r.help||n==="help"){xs(t[1]);return}let s=hr({scope:r.scope}),c=r.store;if(!c)if(r.scope==="project"||r.scope==="local")c=s.jsonStorePath();else c=Ue();if(n==="paths"){y(s.paths(),r.json);return}if(n==="storage"){let a=t[1]??"status";if(a==="status"){let d=s.storageContract(),o=s.validateStorage();y({ok:o.ok,...d,validation:o,message:`${d.storage_type} artifact storage at ${d.artifact_store.uri_prefix}`},r.json);return}if(a==="validate"){let d=s.validateStorage();y({ok:d.ok,validation:d,message:d.ok?"Storage contract valid":`Storage contract invalid: ${d.errors.join("; ")}`},r.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(n==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let o=s.initDb();y({ok:!0,...o,message:`Initialized ${o.path}`},r.json);return}let d=s.dbStats();y({ok:!0,path:s.workspace.knowledgeDbPath,...d,message:`knowledge.db schema v${d.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let d=await s.initWiki();y({ok:!0,...d,message:`Initialized wiki layout in ${s.workspace.home}`},r.json);return}if(n==="safety"){let a=t[1]??"status",d=s.ensureWorkspace(),o=s.safetyPolicy();s.initDb();let l=b(d.knowledgeDbPath);try{if(a==="status"){y({ok:!0,mode:o.mode,workspace:d.home,allow_write_roots:o.allowWriteRoots,read_only_source_access:o.readOnlySourceAccess,network:o.network,redaction:o.redaction,approvals:o.approvals,message:`Safety policy: ${o.mode}`},r.json);return}if(a==="check"){let f=t[2]??"generated_write",g=t[3]??null,E;try{if(f==="web_search")Q(o),E={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!g)throw Error("safety check s3_read requires an s3:// target.");q(g,o),E={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"}}else E=zt(l,o,f,g);A(l,{event_type:"safety_check",action:f,target_uri:g,decision:E.decision==="allow"?"allow":"requires_approval",metadata:E}),y({ok:!0,...E,message:`Safety check ${E.decision}`},r.json);return}catch(h){throw A(l,{event_type:"safety_check",action:f,target_uri:g,decision:"deny",metadata:{error:h instanceof Error?h.message:String(h)}}),h}}if(a==="approve"){let f=t[2]??"generated_write",g=t[3]??null,E=Bt(l,{action:f,target_uri:g,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});A(l,{event_type:"approval",action:f,target_uri:g,decision:"allow",metadata:{approval_id:E.id}}),y({ok:!0,...E,action:f,target_uri:g,message:`Approved ${f}`},r.json);return}if(a==="audit"){let f=l.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((g)=>({id:g.id,event_type:g.event_type,action:g.action,target_uri:g.target_uri,decision:g.decision,metadata:JSON.parse(g.metadata_json),created_at:g.created_at}));y({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(a==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let g=Te(f,o);if(g.findings.length>0)xe(l,{source_uri:"safety://redact",findings:g.findings,metadata:{command:"safety redact"}});A(l,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:g.findings.length>0?"redacted":"allow",metadata:{findings:g.findings.length}}),y({ok:!0,text:g.text,findings:g.findings,message:`Redacted ${g.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{l.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge source resolve <source-ref>");let o=await s.resolveSource(d,{purpose:r.purpose,limit:r.limit});y({ok:!0,...o,message:o.resolved?`Resolved ${o.source_ref} (${o.content.chunks_returned}/${o.content.chunks_total} chunks)`:`Source not indexed: ${d}`},r.json);return}if(n==="ingest"){let a=t[1]??"";if(a==="manifest"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let o=await s.ingestManifest(d);y({ok:!0,...o,message:`Ingested ${o.items_seen} manifest item(s)`},r.json);return}if(a==="source"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest source <source-ref>");let o=await s.ingestSource(d,r.purpose);y({ok:!0,...o,message:`Ingested source ${o.source_ref} (${o.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){let a=t[1]??"status";if(a==="status"){let d=s.reindexHealth({modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`${d.missing_embeddings} chunk(s) missing embeddings`},r.json);return}if(a==="enqueue"){let d=s.enqueueReindex({modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`Queued ${d.enqueued} embedding refresh item(s)`},r.json);return}if(a==="embeddings"){let d=await s.refreshEmbeddings({full:r.full,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`Embedded ${d.indexed.chunks_embedded} chunk(s)`},r.json);return}if(a==="outbox"){let d=t[2];if(!d)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let o=await s.consumeOutbox(d);y({ok:!0,...o,message:`Consumed ${o.events_seen} outbox event(s)`},r.json);return}throw Error("Invalid reindex action. Use 'status', 'enqueue', 'embeddings', or 'outbox'.")}if(n==="embeddings"){let a=t[1]??"status";if(a==="status"){let d=s.embeddingStatus();y({ok:!0,...d,message:`${d.total_vector_entries} vector index entries`},r.json);return}if(a==="index"){let d=await s.indexEmbeddings({limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`Embedded ${d.chunks_embedded} chunk(s)`},r.json);return}if(a==="search"){let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge embeddings search <query>");let o=await s.semanticSearch({query:d,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...o,message:`${o.results.length} semantic result(s)`},r.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(n==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");if(r.context){let o=await s.retrieveContext({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...o,message:`${o.excerpts.length} context excerpt(s)`},r.json);return}let d=await s.search({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`${d.results.length} search result(s)`},r.json);return}if(n==="web"){if((t[1]??"search")!=="search")throw Error("Invalid web action. Use 'search'.");let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge web search <query>");let o=await s.webSearch({query:d,limit:r.limit,modelRef:r.model,provider:r.provider,domains:r.domain,fake:r.fake,fileResults:r.fileResults});y({ok:!0,...o,message:`${o.sources.length} web source(s)`},r.json);return}if(n==="ask"||n==="build"){let a=t.slice(i).join(" ");if(!a)throw Error("Usage: open-knowledge ask <prompt>");let d=await s.runPrompt({prompt:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake,generate:r.generate,approveWrite:r.approveWrite});y({ok:!0,...d,message:d.generated?"Generated answer with citations":"Prepared citation context draft"},r.json);return}if(n==="providers"){let a=t[1]??"status";if(a==="status"){let d=s.providerStatus(),o=d.providers.filter((l)=>l.configured).length;y({ok:!0,...d,message:`${o}/${d.providers.length} provider credential(s) configured`},r.json);return}if(a==="models"){let d=s.modelRegistry();y({ok:!0,models:d,message:`${d.length} model alias(es)`},r.json);return}if(a==="check"){let d=t[2]??"default",o=X(d,s.config()),l=L(o),f=B(l.provider,s.config());y({ok:!0,target:d,model_ref:o,provider:l.provider,model:l.model,credential:f,message:`${l.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(Me(c),n==="add"){let a=t[1],d=t[2];if(!a||!d)throw Error("Usage: open-knowledge add <title> <content>");D(c,()=>{let o=P(c),l={id:je(),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};o.items.push(l),K(c,o),Y("info","Item added",{id:l.id,title:l.title}),y({ok:!0,item:l,message:`Added ${l.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");D(c,()=>{let a=P(c),d=Number.isFinite(r.page)&&r.page>0?r.page:1,o=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,l=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",g=r.format==="table"||!r.json&&!r.format&&Ss(r),E=r.json||r.format==="json",h=a.items;if(r.archived)h=h.filter((v)=>v.archived===!0);else if(!r.includeArchived)h=h.filter((v)=>!v.archived);if(l)h=h.filter((v)=>v.title.toLowerCase().includes(l)||v.content.toLowerCase().includes(l));if(f)h=h.filter((v)=>v.tags&&v.tags.map((Le)=>Le.toLowerCase()).includes(f));let{sorted:m,sort:T,direction:k}=Rs(h,r),p=(d-1)*o,R=m.slice(p,p+o),N=Math.max(1,Math.ceil(m.length/o));if(E){y({ok:!0,page:d,limit:o,total:m.length,total_pages:N,sort:T,direction:k,items:R},!0);return}if(R.length===0){y(`No items found (search=${l||"none"}, tag=${f||"none"})`,!1);return}if(g){let v=(F)=>F,Le=`${v("ID")} ${v("TITLE")} ${v("CREATED")} ${v("URL")} ${v("TAGS")}`;console.log(Le);for(let F of R)console.log(`${F.id} ${v(F.title)} ${F.created_at} ${F.url?v(F.url):""} ${F.tags?.length?v(`[${F.tags.join(", ")}]`):""}`);console.log(`Page ${d}/${N} | showing ${R.length} of ${m.length} | sort=${T} ${k} | search=${l||"none"} | tag=${f||"none"}`)}else{for(let v of R)console.log(`${v.id} ${v.title} ${v.created_at}${v.url?` ${v.url}`:""}${v.tags?.length?` [${v.tags.join(", ")}]`:""}`);console.log(`Page ${d}/${N} | showing ${R.length} of ${m.length} | sort=${T} ${k} | search=${l||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){ce(r),D(c,()=>{let d=P(c).items.find((o)=>o.id===r.id||o.short_id===r.id);if(!d)throw Error(`Item not found: ${r.id}`);y({ok:!0,item:d,message:`${d.id}: ${d.title}`},r.json)});return}if(n==="update"){ce(r),D(c,()=>{let a=P(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];if(r.title!==void 0)o.title=r.title;if(r.content!==void 0)o.content=r.content;if(r.url!==void 0)o.url=r.url;if(r.tag!==void 0){if(o.tags=o.tags||[],!o.tags.map((l)=>l.toLowerCase()).includes(r.tag.toLowerCase()))o.tags.push(r.tag)}o.updated_at=new Date().toISOString(),a.items[d]=o,K(c,a),y({ok:!0,item:o,message:`Updated ${o.id}`},r.json)});return}if(n==="archive"||n==="restore"){ce(r),D(c,()=>{let a=P(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];o.archived=n==="archive",o.updated_at=new Date().toISOString(),a.items[d]=o,K(c,a),y({ok:!0,item:o,message:`${n==="archive"?"Archived":"Restored"} ${o.id}`},r.json)});return}if(n==="untag"){if(ce(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");D(c,()=>{let a=P(c),d=a.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d],l=o.tags?.length??0;o.tags=(o.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),o.updated_at=new Date().toISOString(),a.items[d]=o,K(c,a),y({ok:!0,item:o,removed:l-o.tags.length,message:`Removed tag from ${o.id}`},r.json)});return}if(n==="upsert"){let a=r.title??t[1],d=r.content??t[2];D(c,()=>{let o=P(c),l=r.id?o.items.findIndex((E)=>E.id===r.id||E.short_id===r.id):-1,f=new Date().toISOString();if(l===-1){if(!a||!d)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let E=r.id??je(),h={id:E,short_id:dt(E),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};o.items.push(h),K(c,o),y({ok:!0,created:!0,item:h,message:`Upserted ${h.id}`},r.json);return}let g=o.items[l];if(a!==void 0)g.title=a;if(d!==void 0)g.content=d;if(r.url!==void 0)g.url=r.url;if(r.tag!==void 0){if(g.tags=g.tags||[],!g.tags.map((E)=>E.toLowerCase()).includes(r.tag.toLowerCase()))g.tags.push(r.tag)}g.updated_at=f,o.items[l]=g,K(c,o),y({ok:!0,created:!1,item:g,message:`Upserted ${g.id}`},r.json)});return}if(n==="delete"){if(ce(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");D(c,()=>{let a=P(c),d=a.items.length;a.items=a.items.filter((l)=>l.id!==r.id&&l.short_id!==r.id);let o=d!==a.items.length;if(K(c,a),!o)throw Error(`Item not found: ${r.id}`);Y("info","Item deleted",{id:r.id}),y({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let a=r.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");D(c,()=>{let d=P(c);if(a==="jsonl")for(let o of d.items)console.log(JSON.stringify(o));else y({ok:!0,items:d.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");D(c,()=>{let a=P(c),d=a.items.length;if(r.olderThan!==void 0){let l=new Date;l.setDate(l.getDate()-r.olderThan),a.items=a.items.filter((f)=>new Date(f.created_at)>=l)}if(r.empty)a.items=a.items.filter((l)=>l.content.trim().length>0);let o=d-a.items.length;K(c,a),Y("info","Prune completed",{pruned:o,remaining:a.items.length}),y({ok:!0,pruned:o,remaining:a.items.length,message:`Pruned ${o} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");D(c,()=>{let a=P(c),d=new Set,o=a.items.length;a.items=a.items.filter((f)=>{let g=`${f.title}\x00${f.content}`;if(d.has(g))return!1;return d.add(g),!0});let l=o-a.items.length;K(c,a),Y("info","Dedupe completed",{removed:l,remaining:a.items.length}),y({ok:!0,removed:l,remaining:a.items.length,message:`Dedupe removed ${l} duplicate(s)`},r.json)});return}if(n==="stats"){D(c,()=>{let a=P(c),d=a.items.filter((k)=>!k.archived),o=d.length,l=a.items.length-o,f=d.filter((k)=>k.url).length,g=d.filter((k)=>k.tags&&k.tags.length>0).length,E=o>0?d.map((k)=>k.created_at).sort()[0]:null,h=o>0?d.map((k)=>k.created_at).sort()[o-1]:null,m={};for(let k of d)for(let p of k.tags||[])m[p]=(m[p]||0)+1;let T=Object.entries(m).sort((k,p)=>p[1]-k[1]).slice(0,5).map(([k,p])=>({tag:k,count:p}));y({ok:!0,total:o,archived:l,with_url:f,with_tags:g,oldest:E,newest:h,top_tags:T,message:`${o} items | ${f} with URL | ${g} with tags`},r.json)});return}let u=vs(t[0]),_=u?` Did you mean '${u}'?`:"";throw Y("warn","Unknown command",{input:t[0],suggestion:u}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)Os(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);Y("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{vs as suggestCommand,Rs as sortItems,Os as run,ys as parseArgs};
|