@hasna/knowledge 0.2.17 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/bin/open-knowledge-mcp.js +283 -6
- package/bin/open-knowledge.js +64 -57
- package/docs/architecture/ai-native-knowledge-base.md +7 -0
- package/docs/architecture/hybrid-semantic-search.md +5 -0
- package/package.json +1 -1
- package/src/cli.ts +33 -4
- package/src/mcp.js +18 -0
- package/src/service.ts +11 -0
- package/src/web-search.ts +330 -0
package/bin/open-knowledge.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
|
-
var
|
|
4
|
-
`);return t}function
|
|
3
|
+
var R=import.meta.require;import{readFileSync as le,writeFileSync as de,existsSync as ue,renameSync as wr,unlinkSync as st}from"fs";import{randomUUID as ot}from"crypto";import{existsSync as mr,mkdirSync as Ie,readFileSync as hr,writeFileSync as kr}from"fs";import{homedir as tt}from"os";import{dirname as yr,join as I,resolve as Er}from"path";var te=I(".hasna","apps","knowledge");function Le(){return I(tt(),".open-knowledge","db.json")}function Ce(){return I(tt(),".hasna","apps","knowledge")}function br(e=process.cwd()){return Er(e,te)}function ee(e){return{home:e,configPath:I(e,"config.json"),jsonStorePath:I(e,"db.json"),knowledgeDbPath:I(e,"knowledge.db"),artifactsDir:I(e,"artifacts"),cacheDir:I(e,"cache"),exportsDir:I(e,"exports"),indexesDir:I(e,"indexes"),logsDir:I(e,"logs"),runsDir:I(e,"runs"),schemasDir:I(e,"schemas"),wikiDir:I(e,"wiki")}}function vr(){return{version:1,mode:"local",storage:{type:"local",artifacts_root:"artifacts"},sources:{preferred_ref:"open-files",allowed_schemes:["open-files","s3","file","https","http"]},providers:{default_model:"openai:gpt-5.2",aliases:{fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"},openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},embeddings:{default_model:"openai:text-embedding-3-small",dimensions:1536,batch_size:64,max_parallel_calls:4},safety:{network:{web_search_enabled:!1,s3_reads_enabled:!1,allowed_s3_buckets:[]},redaction:{enabled:!0},approvals:{generated_writes_require_approval:!0}}}}function rt(e){let t=ee(e);Ie(t.home,{recursive:!0});for(let r of[t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir])Ie(r,{recursive:!0});if(!mr(t.configPath))kr(t.configPath,`${JSON.stringify(vr(),null,2)}
|
|
4
|
+
`);return t}function nt(e,t=process.cwd()){if(e==="project"||e==="local")return ee(br(t));return ee(Ce())}function ce(e){Ie(yr(e),{recursive:!0})}function it(e){let t=hr(e,"utf8");return JSON.parse(t)}function Pe(){return ee(Ce()).jsonStorePath}function De(e){if(!ue(e))if(ce(e),e===Pe()&&ue(Le()))de(e,le(Le(),"utf8"));else de(e,JSON.stringify({items:[]},null,2))}function Sr(e){return`${e}.lock`}function Tr(e,t){let i=Date.now();while(Date.now()-i<5000){try{if(!ue(e)){de(e,JSON.stringify({owner:t,ts:Date.now()}));return}let c=JSON.parse(le(e,"utf8"));if(Date.now()-c.ts>1e4)st(e)}catch{}let s=Date.now();while(Date.now()-s<50);}throw Error(`Could not acquire lock on ${e} after 5000ms`)}function xr(e,t){try{if(ue(e)){if(JSON.parse(le(e,"utf8")).owner===t)st(e)}}catch{}}function C(e){De(e);let t=le(e,"utf8"),r=JSON.parse(t);if(!r||!Array.isArray(r.items))return{items:[]};return r}function M(e,t){let r=`${e}.tmp.${ot()}`;de(r,JSON.stringify(t,null,2)),wr(r,e)}function P(e,t){let r=ot(),n=Sr(e);Tr(n,r);try{return t()}finally{xr(n,r)}}function Ue(){return`k_${Date.now().toString(36)}_${Math.random().toString(36).slice(2,8)}`}function at(e){return e.replace(/^k_/,"").slice(0,12)}import{Database as Rr}from"bun:sqlite";var Or=`
|
|
5
5
|
PRAGMA journal_mode = WAL;
|
|
6
6
|
PRAGMA foreign_keys = ON;
|
|
7
7
|
|
|
@@ -168,7 +168,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
168
168
|
|
|
169
169
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
170
170
|
VALUES (1, datetime('now'));
|
|
171
|
-
`,
|
|
171
|
+
`,Nr=`
|
|
172
172
|
DROP TABLE IF EXISTS chunks_fts;
|
|
173
173
|
|
|
174
174
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
@@ -181,7 +181,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
181
181
|
|
|
182
182
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
183
183
|
VALUES (2, datetime('now'));
|
|
184
|
-
`,
|
|
184
|
+
`,Ar=`
|
|
185
185
|
CREATE TABLE IF NOT EXISTS audit_events (
|
|
186
186
|
id TEXT PRIMARY KEY,
|
|
187
187
|
event_type TEXT NOT NULL,
|
|
@@ -212,7 +212,7 @@ CREATE INDEX IF NOT EXISTS idx_approval_gates_status ON approval_gates(status);
|
|
|
212
212
|
|
|
213
213
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
214
214
|
VALUES (3, datetime('now'));
|
|
215
|
-
`,
|
|
215
|
+
`,Ir=`
|
|
216
216
|
CREATE TABLE IF NOT EXISTS vector_index_entries (
|
|
217
217
|
id TEXT PRIMARY KEY,
|
|
218
218
|
chunk_id TEXT NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
|
|
@@ -243,8 +243,8 @@ CREATE INDEX IF NOT EXISTS idx_vector_index_status ON vector_index_entries(statu
|
|
|
243
243
|
|
|
244
244
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
245
245
|
VALUES (4, datetime('now'));
|
|
246
|
-
`;function w(e){
|
|
247
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[r,t.run_id??null,t.provider,t.model,t.input_tokens,t.output_tokens,t.cost_usd,JSON.stringify(t.metadata),t.created_at??new Date().toISOString()]),r}import{createHash as
|
|
246
|
+
`;function w(e){ce(e);let t=new Rr(e);return t.exec("PRAGMA foreign_keys = ON;"),t.exec("PRAGMA busy_timeout = 5000;"),t}function O(e){let t=w(e);try{if(t.exec(Or),re(t)<2)t.exec(Nr);if(re(t)<3)t.exec(Ar);if(re(t)<4)t.exec(Ir);return{path:e,schema_version:re(t)}}finally{t.close()}}function re(e){return e.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0}function L(e,t){return e.query(`SELECT COUNT(*) AS n FROM ${t}`).get()?.n??0}function ct(e){let t=w(e);try{return{schema_version:re(t),sources:L(t,"sources"),source_revisions:L(t,"source_revisions"),chunks:L(t,"chunks"),wiki_pages:L(t,"wiki_pages"),citations:L(t,"citations"),indexes:L(t,"knowledge_indexes"),runs:L(t,"runs"),run_events:L(t,"run_events"),redaction_findings:L(t,"redaction_findings"),audit_events:L(t,"audit_events"),approval_gates:L(t,"approval_gates"),storage_objects:L(t,"storage_objects"),embeddings:L(t,"chunk_embeddings"),vector_entries:L(t,"vector_index_entries")}}finally{t.close()}}import{existsSync as Lr,mkdirSync as dt,readFileSync as Cr,writeFileSync as Pr}from"fs";import{dirname as Dr,join as je,relative as Ur,sep as jr}from"path";function ne(e){let t=e.replace(/\\/g,"/").trim();if(!t||t.startsWith("/"))throw Error(`Invalid artifact key: ${e}`);let r=t.split("/").filter(Boolean);if(r.length===0||r.some((n)=>n==="."||n===".."))throw Error(`Invalid artifact key: ${e}`);return r.join("/")}function Ke(e,t){let r=Ur(e,t);if(r.startsWith("..")||r===".."||r.startsWith(`..${jr}`))throw Error(`Artifact path escapes root: ${t}`)}class ut{root;type="local";canRead=!0;canWrite=!0;constructor(e){this.root=e;dt(e,{recursive:!0})}async put(e){let t=ne(e.key),r=je(this.root,t);return Ke(this.root,r),dt(Dr(r),{recursive:!0}),Pr(r,e.body),{key:t,uri:`file://${r}`}}async getText(e){let t=ne(e),r=je(this.root,t);return Ke(this.root,r),Cr(r,"utf8")}async exists(e){let t=ne(e),r=je(this.root,t);return Ke(this.root,r),Lr(r)}}class lt{options;type="s3";canRead=!0;canWrite=!0;client;constructor(e){this.options=e;this.client=e.client}async getClient(){if(this.client)return this.client;let[{S3Client:e},{fromIni:t}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]);return this.client=new e({region:this.options.region,credentials:this.options.profile?t({profile:this.options.profile}):void 0,maxAttempts:this.options.max_attempts}),this.client}objectKey(e){let t=ne(e),r=this.options.prefix?ne(this.options.prefix):"";return r?`${r}/${t}`:t}async put(e){let[{PutObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e.key);return await r.send(new t({Bucket:this.options.bucket,Key:n,Body:e.body,ContentType:e.content_type,Metadata:e.metadata,ServerSideEncryption:this.options.server_side_encryption,SSEKMSKeyId:this.options.kms_key_id})),{key:n,uri:`s3://${this.options.bucket}/${n}`}}async getText(e){let[{GetObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e),i=await r.send(new t({Bucket:this.options.bucket,Key:n}));if(!i.Body)return"";return await i.Body.transformToString()}async exists(e){let[{HeadObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e);try{return await r.send(new t({Bucket:this.options.bucket,Key:n})),!0}catch(i){let s=i instanceof Error?i.name:"";if(s==="NotFound"||s==="NoSuchKey"||s==="NotFoundError")return!1;throw i}}}function _t(e,t){if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)throw Error("S3 artifact storage requires storage.s3.bucket");return new lt({bucket:e.storage.s3.bucket,prefix:e.storage.s3.prefix,region:e.storage.s3.region,profile:e.storage.s3.profile,max_attempts:e.storage.s3.max_attempts,server_side_encryption:e.storage.s3.server_side_encryption,kms_key_id:e.storage.s3.kms_key_id})}return new ut(t.artifactsDir)}import{randomUUID as jt}from"crypto";import{randomUUID as Kr}from"crypto";var Me={openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},Mr={openai:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!0,reasoning:!0,embeddings:!0},anthropic:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!1,reasoning:!0,embeddings:!1},deepseek:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!1,native_web_search:!1,reasoning:!0,embeddings:!1}},Fr={default:"openai:gpt-5.2",fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"};function gt(e){return e?.providers??{}}function W(e,t){let r=gt(e)[t]??{};return{...Me[t],...r}}function pt(e){let t=gt(e);return{...Fr,...t.default_model?{default:t.default_model}:{},...t.aliases??{}}}function U(e){let[t,...r]=e.split(":"),n=r.join(":");if(t!=="openai"&&t!=="anthropic"&&t!=="deepseek")throw Error(`Unsupported AI provider: ${t}`);if(!n)throw Error(`Invalid model ref: ${e}. Expected provider:model.`);return{provider:t,model:n}}function $(e,t){return pt(t)[e]??e}function Fe(e){let t=pt(e);return Object.entries(t).map(([r,n])=>{let i=U(n);return{alias:r,model_ref:n,provider:i.provider,model:i.model,default:r==="default",capabilities:Mr[i.provider]}})}function mt(e,t=process.env){return Object.keys(Me).map((r)=>{let n=W(e,r),i=Boolean(t[n.api_key_env]);return{provider:r,api_key_env:n.api_key_env,configured:i,source:i?"env":"missing",base_url:n.base_url??null,default_model:n.default_model}})}function ht(e,t=process.env){return{default_model:$("default",e),providers:mt(e,t),models:Fe(e)}}function B(e,t,r=process.env){let n=mt(t,r).find((i)=>i.provider===e);if(!n)throw Error(`Unsupported AI provider: ${e}`);if(!n.configured)throw Error(`Missing ${n.api_key_env} for ${e}. Set the env var to use this provider.`);return n}async function Wr(e){if(e==="openai"){let{createOpenAI:r}=await import("@ai-sdk/openai");return r}if(e==="anthropic"){let{createAnthropic:r}=await import("@ai-sdk/anthropic");return r}let{createDeepSeek:t}=await import("@ai-sdk/deepseek");return t}async function $r(e={}){let{createProviderRegistry:t}=await import("ai"),r=e.env??process.env,n={};for(let i of Object.keys(Me)){let s=W(e.config,i),c=r[s.api_key_env];if(!c)continue;let u=e.factories?.[i]??await Wr(i);n[i]=u({apiKey:c,baseURL:s.base_url})}return t(n)}async function kt(e,t={}){let r=$(e,t.config),n=U(r);return B(n.provider,t.config,t.env),(await $r(t)).languageModel(r)}function ft(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return 0}function _e(e){let t=e.usage??{};return{provider:e.provider,model:e.model,input_tokens:ft(t,["inputTokens","promptTokens","input_tokens","prompt_tokens"]),output_tokens:ft(t,["outputTokens","completionTokens","output_tokens","completion_tokens"]),cost_usd:e.costUsd??0,metadata:{usage:t,provider_metadata:e.providerMetadata??{}}}}function fe(e,t){let r=`usage_${Kr()}`;return e.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
247
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[r,t.run_id??null,t.provider,t.model,t.input_tokens,t.output_tokens,t.cost_usd,JSON.stringify(t.metadata),t.created_at??new Date().toISOString()]),r}import{createHash as un}from"crypto";function We(e){return["deleted","stale","invalidated","reindex_required"].includes((e??"").toLowerCase())}function X(e){let t=e.status??null;return{source_owner:"open-files",source_ref:e.source_ref??null,source_uri:e.source_uri??null,source_kind:e.source_kind??null,source_revision_id:e.source_revision_id??null,revision:e.revision??null,hash:e.hash??null,chunk_id:e.chunk_id??null,start_offset:e.start_offset??null,end_offset:e.end_offset??null,status:t,read_only:!0,citation_required:!0,resolver:e.resolver??null,stale:We(t)}}function $e(e){return{source_owner:"open-files",generated_from:e.generated_from,artifact_key:e.artifact_key,source_refs:e.source_refs??[],read_only_sources:!0,citation_required:e.citation_required??!0,raw_source_bytes_stored_in_open_knowledge:!1}}function yt(e,t){return{...e,provenance:t}}import{createHash as vt}from"crypto";var Xr="openai:text-embedding-3-small",wt=1536;function ge(e){return e?.embeddings??{}}function Et(e,t){return`${e}_${vt("sha256").update(t).digest("hex").slice(0,20)}`}function qe(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function K(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function bt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Xe(e){return Math.sqrt(e.reduce((t,r)=>t+r*r,0))}function qr(e,t,r=Xe(t)){let n=Xe(e);if(n===0||r===0)return 0;let i=Math.min(e.length,t.length),s=0;for(let c=0;c<i;c+=1)s+=e[c]*t[c];return s/(n*r)}function Hr(e,t){let r=vt("sha256").update(e).digest();return Array.from({length:t},(n,i)=>{let s=r[i%r.length]/255;return Number((s*2-1).toFixed(6))})}async function Br(e,t,r=process.env){B("openai",t,r);let n=W(t,"openai"),{createOpenAI:i}=await import("@ai-sdk/openai"),s=i({apiKey:r[n.api_key_env],baseURL:n.base_url});if(s.embeddingModel)return s.embeddingModel(e);if(s.textEmbedding)return s.textEmbedding(e);if(s.textEmbeddingModel)return s.textEmbeddingModel(e);throw Error("OpenAI provider does not expose an embedding model factory.")}function He(e,t){if(!e||e==="default"||e==="embedding")return ge(t).default_model??Xr;return e}async function St(e,t={}){let r=He(t.modelRef,t.config),n=U(r);if(n.provider!=="openai")throw Error(`Embedding provider ${n.provider} is not supported yet. Use openai:text-embedding-3-small.`);let i=t.dimensions??ge(t.config).dimensions??wt;if(t.fake)return{provider:n.provider,model:n.model,dimensions:i,vectors:e.map((a)=>Hr(a,i)),usage:{input_tokens:e.reduce((a,d)=>a+Math.max(1,Math.ceil(d.split(/\s+/).filter(Boolean).length*1.25)),0)}};let{embedMany:s}=await import("ai"),c=await Br(n.model,t.config,t.env),u=await s({model:c,values:e,maxParallelCalls:t.maxParallelCalls??ge(t.config).max_parallel_calls,providerOptions:{openai:{dimensions:i}}}),_=u.embeddings;return{provider:n.provider,model:n.model,dimensions:_[0]?.length??i,vectors:_,usage:{input_tokens:u.usage?.tokens??0}}}function zr(e,t){if(t.sourceRevisionId)return e.query(`SELECT
|
|
248
248
|
c.id,
|
|
249
249
|
c.text,
|
|
250
250
|
c.token_count,
|
|
@@ -282,14 +282,14 @@ VALUES (4, datetime('now'));
|
|
|
282
282
|
ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
|
|
283
283
|
WHERE v.id IS NULL
|
|
284
284
|
ORDER BY c.created_at ASC, c.ordinal ASC
|
|
285
|
-
LIMIT ?`).all(t.provider,t.model,t.limit)}function
|
|
285
|
+
LIMIT ?`).all(t.provider,t.model,t.limit)}function Gr(e){let t=qe(e.metadata_json),r=t.provenance;if(r&&typeof r==="object"&&!Array.isArray(r))return r;return X({source_ref:K(t,["source_ref"]),source_uri:e.source_uri??K(t,["source_uri"]),source_kind:e.source_kind??K(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??K(t,["revision"]),hash:e.hash??K(t,["hash"]),chunk_id:e.id,start_offset:e.start_offset??bt(t,["start_offset"]),end_offset:e.end_offset??bt(t,["end_offset"]),status:K(t,["status"]),resolver:"open-files-read-only"})}function Jr(e,t,r,n){let i=e.prepare(`
|
|
286
286
|
INSERT INTO chunk_embeddings (id, chunk_id, provider, model, dimensions, vector_json, created_at)
|
|
287
287
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
288
288
|
ON CONFLICT(chunk_id, provider, model) DO UPDATE SET
|
|
289
289
|
dimensions = excluded.dimensions,
|
|
290
290
|
vector_json = excluded.vector_json,
|
|
291
291
|
created_at = excluded.created_at
|
|
292
|
-
`),
|
|
292
|
+
`),s=e.prepare(`
|
|
293
293
|
INSERT INTO vector_index_entries (
|
|
294
294
|
id, chunk_id, source_revision_id, provider, model, dimensions, vector_json, vector_norm,
|
|
295
295
|
source_uri, source_ref, revision, hash, start_offset, end_offset, token_count, status,
|
|
@@ -311,10 +311,10 @@ VALUES (4, datetime('now'));
|
|
|
311
311
|
status = excluded.status,
|
|
312
312
|
metadata_json = excluded.metadata_json,
|
|
313
313
|
updated_at = excluded.updated_at
|
|
314
|
-
`);return e.transaction(()=>{for(let u=0;u<t.length;u+=1){let _=t[u],a=r.vectors[u];if(!a)continue;let d=
|
|
314
|
+
`);return e.transaction(()=>{for(let u=0;u<t.length;u+=1){let _=t[u],a=r.vectors[u];if(!a)continue;let d=qe(_.metadata_json),o=Gr(_),l=o.source_ref??K(d,["source_ref"]),f=o.source_uri??_.source_uri??K(d,["source_uri"]),g=o.revision??_.revision??K(d,["revision"]),k=o.hash??_.hash??K(d,["hash"]),h=o.status??K(d,["status"])??"active",m=JSON.stringify(a);i.run(Et("emb",`${_.id}\x00${r.provider}\x00${r.model}`),_.id,r.provider,r.model,r.dimensions,m,n),s.run(Et("vec",`${_.id}\x00${r.provider}\x00${r.model}`),_.id,_.source_revision_id,r.provider,r.model,r.dimensions,m,Xe(a),f,l,g,k,o.start_offset,o.end_offset,_.token_count,h,JSON.stringify({...d,provenance:o,embedded_at:n}),n,n)}})(),t.length}async function Tt(e){let t=He(e.modelRef,e.config),r=U(t);if(r.provider!=="openai")throw Error(`Embedding provider ${r.provider} is not supported yet.`);let n=(e.now??new Date).toISOString(),i=Math.max(1,Math.min(e.limit??100,1000));O(e.dbPath);let s=w(e.dbPath),c;try{c=zr(s,{provider:r.provider,model:r.model,limit:i,sourceRevisionId:e.sourceRevisionId})}finally{s.close()}if(c.length===0)return{provider:r.provider,model:r.model,dimensions:e.dimensions??ge(e.config).dimensions??wt,chunks_seen:0,chunks_embedded:0,embeddings_upserted:0,vector_entries_upserted:0,usage:{input_tokens:0}};let u=await St(c.map((a)=>a.text),e),_=w(e.dbPath);try{let a=Jr(_,c,u,n);return{provider:u.provider,model:u.model,dimensions:u.dimensions,chunks_seen:c.length,chunks_embedded:c.length,embeddings_upserted:a,vector_entries_upserted:a,usage:u.usage}}finally{_.close()}}function xt(e){O(e);let t=w(e);try{let r=t.query("SELECT COUNT(*) AS n FROM chunk_embeddings").get()?.n??0,n=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0,i=t.query(`SELECT provider, model, dimensions, COUNT(*) AS entries, MAX(updated_at) AS updated_at
|
|
315
315
|
FROM vector_index_entries
|
|
316
316
|
GROUP BY provider, model, dimensions
|
|
317
|
-
ORDER BY provider, model`).all();return{total_embeddings:r,total_vector_entries:n,indexes:i}}finally{t.close()}}async function
|
|
317
|
+
ORDER BY provider, model`).all();return{total_embeddings:r,total_vector_entries:n,indexes:i}}finally{t.close()}}async function pe(e){let t=He(e.modelRef,e.config),r=U(t),n=Math.max(1,Math.min(e.limit??10,100)),i=await St([e.query],e),s=i.vectors[0]??[];O(e.dbPath);let c=w(e.dbPath);try{let _=c.query(`SELECT
|
|
318
318
|
v.chunk_id,
|
|
319
319
|
c.text,
|
|
320
320
|
v.vector_json,
|
|
@@ -326,7 +326,7 @@ VALUES (4, datetime('now'));
|
|
|
326
326
|
v.metadata_json
|
|
327
327
|
FROM vector_index_entries v
|
|
328
328
|
JOIN chunks c ON c.id = v.chunk_id
|
|
329
|
-
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(r.provider,r.model).map((a)=>{let d=JSON.parse(a.vector_json),
|
|
329
|
+
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(r.provider,r.model).map((a)=>{let d=JSON.parse(a.vector_json),o=qe(a.metadata_json),l=o.provenance&&typeof o.provenance==="object"&&!Array.isArray(o.provenance)?o.provenance:null;return{chunk_id:a.chunk_id,score:qr(s,d,a.vector_norm),text:a.text,source_uri:a.source_uri,source_ref:a.source_ref,revision:a.revision,hash:a.hash,provenance:l}}).sort((a,d)=>d.score-a.score).slice(0,n);return{provider:r.provider,model:r.model,dimensions:i.dimensions,query:e.query,results:_}}finally{c.close()}}function he(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function D(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function Rt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Ot(e){return Array.from(new Set(e))}function Yr(e){let t=e.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}_]+/gu)??[];return Ot(t.filter((r)=>r.length>0)).slice(0,16)}function Vr(e){if(e.length===0)return null;return e.map((t)=>`${t}*`).join(" OR ")}function Qr(e){return e.replace(/[\\%_]/g,(t)=>`\\${t}`)}function Nt(e,t){return e.flatMap((r)=>Array.from({length:t},()=>`%${Qr(r)}%`))}function Zr(e,t){let r=Number.isFinite(e)?1/(1+Math.abs(e)):0,n=1/(1+t);return ke(Math.max(r,n))}function At(e,t){if(t.length===0)return 0;let r=t.filter((n)=>e.includes(n)).length;if(r===0)return 0;return ke(Math.min(0.85,0.35+r/t.length*0.5))}function en(e){return ke(Math.max(0,Math.min(1,(e+1)/2)))}function ke(e){return Number(e.toFixed(6))}function ie(e,t){let r=e.keyword??0,n=e.semantic??0,i=e.catalog??0,s=t?.chunk_id?0.05:0;return ke(Math.min(1,r*0.55+n*0.4+i*0.35+s))}function Be(e){let t=e.provenance;return t&&typeof t==="object"&&!Array.isArray(t)?t:null}function tn(e){let t=he(e.chunk_metadata_json),r=Be(t);if(r)return r;if(!e.source_revision_id&&!e.source_uri)return null;return X({source_ref:D(t,["source_ref"]),source_uri:e.source_uri??D(t,["source_uri"]),source_kind:e.source_kind??D(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??D(t,["revision"]),hash:e.hash??D(t,["hash"]),chunk_id:e.chunk_id,start_offset:e.start_offset??Rt(t,["start_offset"]),end_offset:e.end_offset??Rt(t,["end_offset"]),status:D(t,["status"]),resolver:"open-files-read-only"})}function rn(e,t,r){if(!t)return[];return e.query(`SELECT
|
|
330
330
|
chunks_fts.chunk_id,
|
|
331
331
|
c.kind AS chunk_kind,
|
|
332
332
|
c.wiki_page_id,
|
|
@@ -355,88 +355,91 @@ VALUES (4, datetime('now'));
|
|
|
355
355
|
LEFT JOIN wiki_pages wp ON wp.id = c.wiki_page_id
|
|
356
356
|
WHERE chunks_fts MATCH ?
|
|
357
357
|
ORDER BY rank ASC
|
|
358
|
-
LIMIT ?`).all(t,r)}function
|
|
358
|
+
LIMIT ?`).all(t,r)}function It(e,t){if(t.length===0)return"1 = 0";return t.map(()=>`(${e.map((n)=>`lower(COALESCE(${n}, '')) LIKE ? ESCAPE '\\'`).join(" OR ")})`).join(" OR ")}function nn(e,t,r){let n=["path","title","artifact_uri","metadata_json"];return e.query(`SELECT id, path, title, artifact_uri, content_hash, status, metadata_json
|
|
359
359
|
FROM wiki_pages
|
|
360
|
-
WHERE status = 'active' AND (${
|
|
360
|
+
WHERE status = 'active' AND (${It(n,t)})
|
|
361
361
|
ORDER BY updated_at DESC
|
|
362
|
-
LIMIT ?`).all(...
|
|
362
|
+
LIMIT ?`).all(...Nt(t,n.length),r)}function sn(e,t,r){let n=["kind","name","shard_key","artifact_uri","metadata_json"];return e.query(`SELECT id, kind, name, artifact_uri, shard_key, metadata_json
|
|
363
363
|
FROM knowledge_indexes
|
|
364
|
-
WHERE ${
|
|
364
|
+
WHERE ${It(n,t)}
|
|
365
365
|
ORDER BY updated_at DESC
|
|
366
|
-
LIMIT ?`).all(...
|
|
366
|
+
LIMIT ?`).all(...Nt(t,n.length),r)}function on(e,t){let r=he(e.chunk_metadata_json),n=tn(e),i=D(r,["source_ref"]),s=e.source_uri??D(r,["source_uri"]),c=Boolean(e.wiki_page_id),u={kind:c?"wiki_chunk":"source_chunk",id:e.chunk_id,title:c?e.wiki_title:e.source_title,text:e.text,score:0,scores:{keyword:t},source:s||i?{uri:s,ref:i,kind:e.source_kind??D(r,["source_kind"]),revision:e.revision??D(r,["revision"]),hash:e.hash??D(r,["hash"])}:null,citation:{chunk_id:e.chunk_id,start_offset:e.start_offset,end_offset:e.end_offset},artifact:c?{uri:e.wiki_artifact_uri,path:e.wiki_path,hash:e.wiki_content_hash,shard_key:e.wiki_path}:null,provenance:n,reasons:["keyword_match"]};return u.score=ie(u.scores,u.citation),u}function an(e,t){let r=he(e.metadata_json),n=At(`${e.path} ${e.title} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"wiki_page",id:e.id,title:e.title,text:null,score:0,scores:{catalog:n},source:null,citation:null,artifact:{uri:e.artifact_uri,path:e.path,hash:e.content_hash,shard_key:e.path},provenance:Be(r),reasons:["wiki_catalog_match"]};return i.score=ie(i.scores,i.citation),i}function cn(e,t){let r=he(e.metadata_json),n=At(`${e.kind} ${e.name} ${e.shard_key??""} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"knowledge_index",id:e.id,title:e.name,text:null,score:0,scores:{catalog:n},source:null,citation:null,artifact:{uri:e.artifact_uri,path:D(r,["artifact_key"]),hash:D(r,["content_hash"]),shard_key:e.shard_key},provenance:Be(r),reasons:["index_catalog_match"]};return i.score=ie(i.scores,i.citation),i}function me(e,t){let r=`${t.kind}:${t.id}`,n=e.get(r);if(!n){e.set(r,t);return}n.scores={keyword:Math.max(n.scores.keyword??0,t.scores.keyword??0)||void 0,semantic:Math.max(n.scores.semantic??0,t.scores.semantic??0)||void 0,catalog:Math.max(n.scores.catalog??0,t.scores.catalog??0)||void 0},n.reasons=Ot([...n.reasons,...t.reasons]),n.text=n.text??t.text,n.title=n.title??t.title,n.source=n.source??t.source,n.citation=n.citation??t.citation,n.artifact=n.artifact??t.artifact,n.provenance=n.provenance??t.provenance,n.score=ie(n.scores,n.citation)}function dn(e){let t={source_chunk:0,wiki_chunk:1,wiki_page:2,knowledge_index:3};return e.sort((r,n)=>{if(n.score!==r.score)return n.score-r.score;return t[r.kind]-t[n.kind]||r.id.localeCompare(n.id)})}async function ye(e){let t=e.query.trim();if(!t)throw Error("Search query is required.");let r=Math.max(1,Math.min(e.limit??10,100)),n=Yr(t),i=Vr(n),s=e.semantic===!0||e.fake===!0||Boolean(e.modelRef),c=[],u=null,_=null,a=null,d=0,o=0,l=0,f=new Map;O(e.dbPath);let g=w(e.dbPath);try{let h=rn(g,i,Math.max(r*3,20));d=h.length,h.forEach((y,p)=>me(f,on(y,Zr(y.rank,p))));let m=nn(g,n,Math.max(r,10)),S=sn(g,n,Math.max(r,10));o=m.length+S.length,m.forEach((y)=>me(f,an(y,n))),S.forEach((y)=>me(f,cn(y,n)))}finally{g.close()}if(s)try{let h=await pe({dbPath:e.dbPath,query:t,limit:Math.max(r*3,20),config:e.config,env:e.env,modelRef:e.modelRef,dimensions:e.dimensions,fake:e.fake,batchSize:e.batchSize,maxParallelCalls:e.maxParallelCalls});u=h.provider,_=h.model,a=h.dimensions,l=h.results.length;for(let m of h.results){let S={kind:"source_chunk",id:m.chunk_id,title:null,text:m.text,score:0,scores:{semantic:en(m.score)},source:{uri:m.source_uri,ref:m.source_ref,kind:m.provenance?.source_kind??null,revision:m.revision,hash:m.hash},citation:{chunk_id:m.chunk_id,start_offset:m.provenance?.start_offset??null,end_offset:m.provenance?.end_offset??null},artifact:null,provenance:m.provenance,reasons:["semantic_match"]};S.score=ie(S.scores,S.citation),me(f,S)}}catch(h){c.push(`semantic_search_failed: ${h instanceof Error?h.message:String(h)}`)}let k=dn(Array.from(f.values())).slice(0,r);return{query:t,limit:r,mode:{keyword:!0,catalog:!0,semantic:s},semantic_provider:u,semantic_model:_,semantic_dimensions:a,counts:{keyword_results:d,catalog_results:o,semantic_results:l,merged_results:k.length},warnings:c,results:k}}function Lt(e,t){return`${e}_${un("sha256").update(t).digest("hex").slice(0,20)}`}function Ct(e){return e.normalize("NFKC").trim().replace(/\s+/g," ").toLowerCase()}function ln(e){return Array.from(new Set(Ct(e).match(/[\p{L}\p{N}_]+/gu)??[])).slice(0,16)}function _n(e){return[e.title,e.text].filter(Boolean).join(" ").toLowerCase()}function fn(e,t){if(t.length===0)return 0;let r=_n(e),n=t.filter((i)=>r.includes(i)).length;return Number((n/t.length).toFixed(6))}function gn(e){if(!e)return!0;if("read_only"in e)return e.read_only===!0;if("read_only_sources"in e)return e.read_only_sources===!0;return!0}function Pt(e){if(!e)return!1;if("stale"in e&&e.stale)return!0;if("status"in e)return We(e.status);return!1}function pn(e){if(Pt(e.provenance))return 0;if(e.source?.hash||e.source?.revision)return 1;if(e.artifact?.hash)return 0.85;if(e.provenance&&"source_refs"in e.provenance&&e.provenance.source_refs.length>0)return 0.75;return 0.55}function mn(e){if(e.citation?.chunk_id&&(e.source?.uri||e.artifact?.uri))return 1;if(e.provenance&&"citation_required"in e.provenance&&e.provenance.citation_required)return 0.75;if(e.artifact?.uri)return 0.65;return 0.35}function hn(e){if(e.kind==="wiki_chunk")return 0.85;if(e.kind==="source_chunk")return 0.8;if(e.kind==="wiki_page")return 0.65;return 0.55}function kn(e,t){let r={base_score:e.score,exact_score:fn(e,t),citation_score:mn(e),freshness_score:pn(e),authority_score:hn(e)},n=Math.min(1,r.base_score*0.65+r.exact_score*0.1+r.citation_score*0.1+r.freshness_score*0.1+r.authority_score*0.05),i=new Set(e.reasons);if(r.exact_score>0.5)i.add("exact_term");if(r.citation_score>=0.75)i.add("cited_source");if(r.freshness_score>=0.85)i.add("fresh_source");return{...e,score:Number(n.toFixed(6)),reasons:Array.from(i),rerank:{...r,final_score:Number(n.toFixed(6))}}}function Dt(e,t){let r=e.text??e.title;if(!r)return null;let n=r.replace(/\s+/g," ").trim();return n.length<=t?n:`${n.slice(0,Math.max(0,t-1)).trim()}...`}function yn(e){return{id:Lt("cite",`${e.kind}\x00${e.id}\x00${e.source?.uri??""}\x00${e.artifact?.uri??""}`),result_id:e.id,kind:e.kind,source_uri:e.source?.uri??null,source_ref:e.source?.ref??null,artifact_uri:e.artifact?.uri??null,artifact_path:e.artifact?.path??null,revision:e.source?.revision??null,hash:e.source?.hash??e.artifact?.hash??null,chunk_id:e.citation?.chunk_id??null,start_offset:e.citation?.start_offset??null,end_offset:e.citation?.end_offset??null,quote:Dt(e,500),provenance:e.provenance}}function En(e,t,r){let n=Dt(e,r);if(!n)return null;return{id:Lt("excerpt",`${e.kind}\x00${e.id}`),result_id:e.id,citation_id:t.id,kind:e.kind,text:n,score:e.score}}function Ee(e){return e.map(()=>"?").join(", ")}function bn(e,t){let r=t.map((u)=>u.citation?.chunk_id).filter((u)=>Boolean(u)),n=t.filter((u)=>u.kind==="wiki_page").map((u)=>u.id),i=[],s=[];if(r.length===0&&n.length===0)return{citations:i,backlinks:s};let c=w(e);try{if(r.length>0)i.push(...c.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
367
367
|
FROM citations
|
|
368
|
-
WHERE chunk_id IN (${
|
|
368
|
+
WHERE chunk_id IN (${Ee(r)})
|
|
369
369
|
ORDER BY created_at DESC
|
|
370
370
|
LIMIT 50`).all(...r));if(n.length>0)i.push(...c.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
371
371
|
FROM citations
|
|
372
|
-
WHERE wiki_page_id IN (${
|
|
372
|
+
WHERE wiki_page_id IN (${Ee(n)})
|
|
373
373
|
ORDER BY created_at DESC
|
|
374
|
-
LIMIT 50`).all(...n)),
|
|
374
|
+
LIMIT 50`).all(...n)),s.push(...c.query(`SELECT from_page_id, to_page_id, label
|
|
375
375
|
FROM wiki_backlinks
|
|
376
|
-
WHERE from_page_id IN (${
|
|
377
|
-
LIMIT 50`).all(...n,...n))}finally{c.close()}return{citations:i,backlinks:
|
|
378
|
-
`)}function
|
|
376
|
+
WHERE from_page_id IN (${Ee(n)}) OR to_page_id IN (${Ee(n)})
|
|
377
|
+
LIMIT 50`).all(...n,...n))}finally{c.close()}return{citations:i,backlinks:s}}async function be(e){let t=Math.max(200,Math.min(e.contextChars??1200,4000)),r=await ye(e),n=ln(r.query),i=[...r.warnings],s=new Set,c=new Set,_=r.results.filter((o)=>{if(!gn(o.provenance))return i.push(`permission_filtered: ${o.kind}:${o.id}`),s.add("Dropped a result because provenance was not read-only."),!1;if(Pt(o.provenance))return i.push(`stale_filtered: ${o.kind}:${o.id}`),c.add("Dropped a stale result whose source status requires reindexing."),!1;return!0}).map((o)=>kn(o,n)).sort((o,l)=>l.score-o.score||o.id.localeCompare(l.id)).slice(0,r.limit),a=_.map(yn),d=_.map((o,l)=>En(o,a[l],t)).filter((o)=>Boolean(o));for(let o of _){if(o.provenance&&"read_only"in o.provenance&&o.provenance.read_only)s.add("All source-backed excerpts are read-only and citation-required.");if(o.rerank.freshness_score>=0.85)c.add("Fresh source revision/hash or artifact hash is present for top context.")}return{query:r.query,normalized_query:Ct(r.query),created_at:new Date().toISOString(),mode:r.mode,warnings:i,search_counts:r.counts,results:_,citations:a,excerpts:d,graph:bn(e.dbPath,_),notes:{permissions:Array.from(s),freshness:Array.from(c)}}}function ze(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Je(e){return`C${e+1}`}function vn(e,t){if(t.excerpts.length===0)return`No indexed knowledge matched the prompt: ${e}`;return[`Found ${t.excerpts.length} relevant knowledge excerpt(s) for: ${e}`,"",...t.excerpts.slice(0,5).map((n,i)=>{let s=t.citations.find((u)=>u.id===n.citation_id),c=s?.source_ref??s?.source_uri??s?.artifact_path??s?.artifact_uri??"unknown source";return`[${Je(i)}] ${n.text} (${c})`})].join(`
|
|
378
|
+
`)}function wn(e,t){let r=t.citations.map((i,s)=>({id:Je(s),source_ref:i.source_ref,source_uri:i.source_uri,artifact_path:i.artifact_path,revision:i.revision,hash:i.hash,quote:i.quote})),n=t.excerpts.map((i,s)=>({id:Je(s),kind:i.kind,text:i.text,score:i.score}));return[`Prompt: ${e}`,"","Use only the provided context. Cite claims with citation ids like [C1]. If context is insufficient, say what is missing.","",`Context excerpts:
|
|
379
379
|
${JSON.stringify(n,null,2)}`,"",`Citations:
|
|
380
380
|
${JSON.stringify(r,null,2)}`].join(`
|
|
381
|
-
`)}function
|
|
382
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[t.runId,"knowledge-prompt",t.prompt,t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.now])}finally{r.close()}}function
|
|
383
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${
|
|
381
|
+
`)}function Sn(e,t){if(t.citations.length===0)return[];return[{kind:"answer_note",title:e.length>80?`${e.slice(0,77)}...`:e,citations:t.citations.map((r)=>r.id),requires_approval:!0}]}function Tn(e,t){let r=w(e);try{r.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
382
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[t.runId,"knowledge-prompt",t.prompt,t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.now])}finally{r.close()}}function Ge(e,t){let r=w(e);try{r.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
383
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${jt()}`,t.runId,t.level,t.event,JSON.stringify(t.metadata),t.now])}finally{r.close()}}function Ut(e,t){let r=w(e);try{r.run(`UPDATE runs
|
|
384
384
|
SET status = ?, provider = ?, model = ?, metadata_json = ?, updated_at = ?
|
|
385
|
-
WHERE id = ?`,[t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.runId])}finally{r.close()}}function
|
|
385
|
+
WHERE id = ?`,[t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.runId])}finally{r.close()}}function xn(e,t,r,n,i,s,c={}){let u=w(e);try{fe(u,{run_id:t,provider:n,model:i,input_tokens:r.input_tokens,output_tokens:r.output_tokens,cost_usd:r.cost_usd,metadata:c,created_at:s})}finally{u.close()}}async function Kt(e){let t=e.prompt.trim();if(!t)throw Error("Knowledge prompt is required.");let r=(e.now??new Date).toISOString(),n=`run_${jt()}`,i=$(e.modelRef??"default",e.config),s=U(i);O(e.dbPath),Tn(e.dbPath,{runId:n,prompt:t,status:e.generate?"running":"dry_run",provider:e.generate?s.provider:"local",model:e.generate?s.model:"context-draft",metadata:{semantic:e.semantic===!0||e.fake===!0||Boolean(e.modelRef),approve_write:e.approveWrite===!0,generated:e.generate===!0},now:r});let{prompt:c,generate:u,approveWrite:_,now:a,...d}=e,o=await be({...d,query:t});Ge(e.dbPath,{runId:n,level:"info",event:"context_retrieved",metadata:{results:o.results.length,citations:o.citations.length,warnings:o.warnings},now:r});let l=vn(t,o),f=!1,g="local",k="context-draft",h={input_tokens:ze(t)+o.excerpts.reduce((p,x)=>p+ze(x.text),0),output_tokens:ze(l),cost_usd:0},m=[...o.warnings];if(e.generate)try{if(e.fake)f=!0,g=s.provider,k=s.model,l=`Fake generated answer for: ${t}
|
|
386
386
|
|
|
387
|
-
${l}`;else{let{generateText:p}=await import("ai"),
|
|
388
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function
|
|
389
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${
|
|
390
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function
|
|
387
|
+
${l}`;else{let{generateText:p}=await import("ai"),x=await kt(i,{config:e.config,env:e.env}),N=await p({model:x,system:"You answer company knowledge-base prompts using only provided context and citation ids.",prompt:wn(t,o)});f=!0,g=s.provider,k=s.model,l=N.text;let b=_e({provider:g,model:k,usage:N.usage,providerMetadata:N.providerMetadata});h={input_tokens:b.input_tokens,output_tokens:b.output_tokens,cost_usd:b.cost_usd}}}catch(p){throw Ge(e.dbPath,{runId:n,level:"error",event:"answer_generation_failed",metadata:{message:p instanceof Error?p.message:String(p)},now:r}),Ut(e.dbPath,{runId:n,status:"failed",provider:s.provider,model:s.model,metadata:{generated:!1,error:p instanceof Error?p.message:String(p)},now:r}),p}let S=Sn(t,o),y={approved:e.approveWrite===!0,durable_writes_performed:!1,reason:e.approveWrite?"Approval flag recorded; durable wiki writing is deferred to the wiki compile task.":"Dry-run mode: proposed wiki updates require approval before durable writes."};return Ge(e.dbPath,{runId:n,level:"info",event:f?"answer_generated":"answer_drafted",metadata:{provider:g,model:k,proposed_updates:S.length,durable_writes_performed:!1},now:r}),xn(e.dbPath,n,h,g,k,r,{generated:f,citations:o.citations.length}),Ut(e.dbPath,{runId:n,status:f?"completed":"dry_run",provider:g,model:k,metadata:{generated:f,citations:o.citations.length,proposed_updates:S.length,approve_write:e.approveWrite===!0},now:r}),{run_id:n,prompt:t,generated:f,provider:g,model:k,answer:l,context:o,citations:o.citations,proposed_wiki_updates:S,write_policy:y,usage:h,warnings:m}}import{createHash as Kn,randomUUID as Mn}from"crypto";import{existsSync as Fn,readFileSync as Wn}from"fs";import{basename as $n}from"path";function Mt(e,t){if(!e)throw Error(t);return e}function Rn(e){let r=e.slice(13).split("/").filter(Boolean),n=r[0];if(n!=="file"&&n!=="source")throw Error("Invalid open-files ref. Expected open-files://file/<id>, open-files://file/<id>/revision/<revision_id>, or open-files://source/<id>/path/<path>.");let i=Mt(r[1],"Invalid open-files ref. Missing id.");if(n==="file"){if(r.length===2)return{kind:"open-files",uri:e,entity:n,id:i};if(r[2]==="revision"&&r[3]&&r.length===4)return{kind:"open-files",uri:e,entity:n,id:i,revision_id:decodeURIComponent(r[3])};throw Error("Invalid open-files file ref. Expected open-files://file/<id>/revision/<revision_id>.")}let s=r.indexOf("path"),c=s>=0?decodeURIComponent(r.slice(s+1).join("/")):void 0;return{kind:"open-files",uri:e,entity:n,id:i,path:c}}function On(e){let t=new URL(e),r=Mt(t.hostname,"Invalid s3 ref. Missing bucket."),n=decodeURIComponent(t.pathname.replace(/^\/+/,""));if(!n)throw Error("Invalid s3 ref. Missing object key.");return{kind:"s3",uri:e,bucket:r,key:n}}function Nn(e){let t=new URL(e);return{kind:"file",uri:e,path:decodeURIComponent(t.pathname)}}function An(e){let t=new URL(e);return{kind:"web",uri:e,url:t.toString()}}function j(e){if(e.startsWith("open-files://"))return Rn(e);if(e.startsWith("s3://"))return On(e);if(e.startsWith("file://"))return Nn(e);if(e.startsWith("https://")||e.startsWith("http://"))return An(e);throw Error(`Unsupported source ref scheme: ${e}`)}function Ft(e,t=j(e)){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Wt(e){let t=j(e);return t.kind==="open-files"&&t.entity==="file"?t.revision_id??null:null}import{createHash as In,randomUUID as Ye}from"crypto";import{relative as Ln,resolve as Xt,sep as Cn}from"path";function $t(e){let t=process.env[e];return t==="1"||t==="true"||t==="yes"}function qt(e,t){let r=e,n=new Set(r.safety?.network?.allowed_s3_buckets??[]);if(e.storage.type==="s3"&&e.storage.s3?.bucket)n.add(e.storage.s3.bucket);if(process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS)for(let i of process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.split(",").map((s)=>s.trim()).filter(Boolean))n.add(i);return{mode:e.mode,allowWriteRoots:[t.home,t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir].map((i)=>Xt(i)),readOnlySourceAccess:!0,network:{webSearchEnabled:r.safety?.network?.web_search_enabled??$t("HASNA_KNOWLEDGE_WEB_SEARCH"),s3ReadsEnabled:r.safety?.network?.s3_reads_enabled??$t("HASNA_KNOWLEDGE_ALLOW_S3_READS"),allowedS3Buckets:[...n].sort()},redaction:{enabled:r.safety?.redaction?.enabled??!0},approvals:{generatedWritesRequireApproval:r.safety?.approvals?.generated_writes_require_approval??!0}}}function Pn(e,t){let r=Ln(e,t);return r===""||!r.startsWith("..")&&r!==".."&&!r.startsWith(`..${Cn}`)}function z(e,t){let r=Xt(e);if(!t.allowWriteRoots.some((n)=>Pn(n,r)))throw Error(`Safety policy denied write outside .hasna/apps/knowledge: ${e}`)}function q(e,t){let n=new URL(e).hostname;if(!t.network.s3ReadsEnabled)throw Error("Safety policy denied S3 read. Set safety.network.s3_reads_enabled=true or HASNA_KNOWLEDGE_ALLOW_S3_READS=1.");if(!t.network.allowedS3Buckets.includes(n))throw Error(`Safety policy denied S3 bucket "${n}". Add it to safety.network.allowed_s3_buckets or HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.`)}function Y(e){if(!e.network.webSearchEnabled)throw Error("Safety policy denied web search. Set safety.network.web_search_enabled=true or HASNA_KNOWLEDGE_WEB_SEARCH=1.")}var Dn=[{type:"private_key_block",severity:"high",regex:/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,replacement:"[REDACTED:private_key_block]"},{type:"secret_assignment",severity:"high",regex:/\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*['"]?[^'"\s]{8,}/gi,replacement:"[REDACTED:secret_assignment]"},{type:"openai_api_key",severity:"high",regex:/\bsk-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:openai_api_key]"},{type:"anthropic_api_key",severity:"high",regex:/\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:anthropic_api_key]"},{type:"aws_access_key_id",severity:"high",regex:/\bA(?:KIA|SIA)[A-Z0-9]{16}\b/g,replacement:"[REDACTED:aws_access_key_id]"}];function ve(e,t){if(t&&!t.redaction.enabled)return{text:e,findings:[]};let r=e,n=[];for(let i of Dn)r=r.replace(i.regex,(s,...c)=>{let u=typeof c.at(-2)==="number"?c.at(-2):r.indexOf(s);return n.push({type:i.type,severity:i.severity,start:Math.max(0,u),end:Math.max(0,u+s.length)}),i.replacement});return{text:r,findings:n}}function Un(e){return`audit_${In("sha256").update(`${e.event_type}\x00${e.action}\x00${e.target_uri??""}\x00${e.created_at??""}\x00${JSON.stringify(e.metadata??{})}\x00${Ye()}`).digest("hex").slice(0,24)}`}function A(e,t){let r=t.created_at??new Date().toISOString(),n=Un({...t,created_at:r});return e.run(`INSERT INTO audit_events (id, event_type, action, target_uri, decision, metadata_json, created_at)
|
|
388
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function we(e,t){let r=t.created_at??new Date().toISOString();for(let n of t.findings)e.run(`INSERT INTO redaction_findings (id, source_uri, run_id, severity, finding_type, metadata_json, created_at)
|
|
389
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${Ye()}`,t.source_uri??null,t.run_id??null,n.severity,n.type,JSON.stringify({...t.metadata??{},start:n.start,end:n.end}),r]);return t.findings.length}function Ht(e,t){let r=t.created_at??new Date().toISOString(),n=`approval_${Ye()}`;return e.run(`INSERT INTO approval_gates (id, action, target_uri, status, reason, approved_by, metadata_json, created_at, updated_at)
|
|
390
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function jn(e,t,r){let n=e.query(`SELECT id FROM approval_gates
|
|
391
391
|
WHERE action = ? AND status = 'approved' AND (target_uri IS NULL OR target_uri = ? OR ? IS NULL)
|
|
392
|
-
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function
|
|
392
|
+
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function Bt(e,t,r,n){let i=r==="generated_write"&&t.approvals.generatedWritesRequireApproval,s=!i||jn(e,r,n);return{action:r,target_uri:n??null,approval_required:i,approved:s,decision:s?"allow":"requires_approval"}}function Se(e,t){return`${e}_${Kn("sha256").update(t).digest("hex").slice(0,20)}`}function V(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function T(e){return typeof e==="string"&&e.length>0?e:void 0}function Xn(e){let t=T(e.source_ref)??T(e.source_uri)??T(e.uri);if(t)return t;let r=T(e.file_id);if(r){let s=T(e.revision_id)??T(e.revision),c=`open-files://file/${encodeURIComponent(r)}`;return s?`${c}/revision/${encodeURIComponent(s)}`:c}let n=T(e.source_id),i=T(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Outbox event is missing source_ref, file_id, or source_id/path.")}function qn(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Hn(e){return T(e.hash)??T(e.checksum)??T(e.sha256)??null}function Bn(e,t,r){return T(e.revision_id)??T(e.revision)??T(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??null}function zn(e){return(T(e.event)??T(e.type)??T(e.action)??T(e.change_type)??"changed").toLowerCase()}function Gn(e){let t=T(e.path);return T(e.title)??T(e.name)??(t?$n(t):null)}function Jn(e,t){let r=Xn(e),n=j(r),i=Hn(e);return{raw:e,eventType:zn(e),sourceRef:r,sourceUri:qn(r,n),kind:n.kind,title:Gn(e),revision:Bn(e,n,i),hash:i,status:T(e.status)?.toLowerCase()??null,updatedAt:T(e.updated_at)??t,acl:e.permissions??e.acl??void 0}}function Yn(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Outbox array parse failed.");return r.map((n)=>{let i=V(n);if(!i)throw Error("Outbox array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=V(r);if(!n)throw Error("Outbox object parse failed.");if(Array.isArray(n.events))return n.events.map((i)=>{let s=V(i);if(!s)throw Error("Outbox events entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=V(JSON.parse(i));if(!s)throw Error("Outbox JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=V(JSON.parse(r));if(!n)throw Error("Outbox JSONL entries must be objects.");return n})}async function Vn(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 outbox URI: ${e}`);if(r)q(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function Qn(e,t,r){if(e.startsWith("s3://"))return Vn(e,t,r);if(!Fn(e))throw Error(`Outbox not found: ${e}`);return Wn(e,"utf8")}function zt(e,t){let r={};if(e)try{r=V(JSON.parse(e))??{}}catch{r={}}return JSON.stringify({...r,...t})}function Zn(e,t,r){let n=Se("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
393
393
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
394
394
|
ON CONFLICT(uri) DO UPDATE SET
|
|
395
395
|
kind = excluded.kind,
|
|
396
396
|
title = COALESCE(excluded.title, sources.title),
|
|
397
|
-
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let
|
|
397
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(T(t.raw.path))s.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[zt(i.metadata_json,s),t.acl===void 0?null:JSON.stringify(t.acl),t.acl===void 0?null:JSON.stringify(t.acl),t.updatedAt,i.id]),i.id}function ei(e,t,r,n){if(!r.revision)return null;let i=Se("rev",`${t}\x00${r.revision}`),s={source_ref:r.sourceRef,source_uri:r.sourceUri,status:r.status,last_outbox_event:r.eventType,reindex_required:!0};return e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
398
398
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
399
399
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
400
400
|
hash = COALESCE(excluded.hash, source_revisions.hash),
|
|
401
|
-
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,T(r.raw.extracted_text_ref)??null,JSON.stringify(
|
|
402
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
403
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[
|
|
404
|
-
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[
|
|
405
|
-
`);if(!n.trim())return[];let i=[],
|
|
401
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,T(r.raw.extracted_text_ref)??null,JSON.stringify(s),n]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision)?.id??null}function ti(e,t,r){if(r.revision)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").all(t,r.revision).map((n)=>n.id);if(r.hash)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND hash = ?").all(t,r.hash).map((n)=>n.id);return e.query("SELECT id FROM source_revisions WHERE source_id = ?").all(t).map((n)=>n.id)}function ri(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t),n=0,i=0;for(let c of r){let u=e.query("SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?").get(c.id);n+=u?.n??0;let _=e.query("SELECT COUNT(*) AS n FROM vector_index_entries WHERE chunk_id = ?").get(c.id);i+=_?.n??0,e.run("DELETE FROM vector_index_entries WHERE chunk_id = ?",[c.id]),e.run("DELETE FROM chunk_embeddings WHERE chunk_id = ?",[c.id]),e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[c.id])}e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]);let s=e.query("SELECT metadata_json FROM source_revisions WHERE id = ?").get(t);return e.run("UPDATE source_revisions SET metadata_json = ? WHERE id = ?",[zt(s?.metadata_json,{reindex_required:!0,invalidated_at:new Date().toISOString()}),t]),{chunksDeleted:r.length,embeddingsDeleted:n,vectorEntriesDeleted:i}}function ni(e,t){return t==="deleted"||["delete","deleted","remove","removed"].includes(e)}function ii(e){return["move","moved","rename","renamed","path_changed"].includes(e)}function si(e){return["permission","permissions","permission_changed","acl_changed"].includes(e)}async function Gt(e){let t=(e.now??new Date).toISOString();if(e.safetyPolicy)z(e.dbPath,e.safetyPolicy);O(e.dbPath);let r=await Qn(e.input,e.config,e.safetyPolicy),n=Yn(r),i=w(e.dbPath),s=`run_${Mn()}`;try{return i.transaction(()=>{i.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
402
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:n.length}),t,t]);let c=new Set,u=new Set,_=0,a=0,d=0,o=0,l=0,f=0,g=0;return A(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:n.length,read_only:!0},created_at:t}),n.forEach((k,h)=>{let m=Jn(k,t),S=Zn(i,m,t);c.add(S);let y=ei(i,S,m,t);if(y)u.add(y);let p=ti(i,S,m);for(let x of p){u.add(x);let N=ri(i,x);_+=N.chunksDeleted,a+=N.embeddingsDeleted,d+=N.vectorEntriesDeleted,o+=1}if(ni(m.eventType,m.status))l+=1;if(ii(m.eventType))f+=1;if(si(m.eventType)||m.acl!==void 0)g+=1;i.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
403
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[Se("evt",`${s}\x00${h}\x00${m.sourceRef}\x00${m.eventType}`),s,"info",m.eventType,JSON.stringify({source_ref:m.sourceRef,source_uri:m.sourceUri,revision:m.revision,hash:m.hash,status:m.status,affected_revisions:p.length}),m.updatedAt])}),i.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
404
|
+
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[Se("usage",s),s,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),A(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:s,events:n.length,sources:c.size,revisions:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:s,events_seen:n.length,sources_touched:c.size,revisions_touched:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d,stale_revisions:o,deleted_sources:l,moved_sources:f,permission_updates:g}})()}finally{i.close()}}import{createHash as oi}from"crypto";import{existsSync as ai,readFileSync as ci}from"fs";import{basename as di}from"path";function Ve(e,t){return`${e}_${oi("sha256").update(t).digest("hex").slice(0,20)}`}function Q(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function v(e){return typeof e==="string"&&e.length>0?e:void 0}function ui(e){return typeof e==="number"&&Number.isFinite(e)?e:void 0}function li(e){let t=v(e.source_ref)??v(e.source_uri)??v(e.uri);if(t)return t;let r=v(e.file_id);if(r){let s=v(e.revision_id)??v(e.revision),c=`open-files://file/${encodeURIComponent(r)}`;return s?`${c}/revision/${encodeURIComponent(s)}`:c}let n=v(e.source_id),i=v(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Manifest item is missing source_ref, file_id, or source_id/path.")}function _i(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function fi(e){let t=v(e.extracted_text)??v(e.text)??v(e.content_text)??v(e.markdown);if(t!==void 0)return t;let r=e.content;return typeof r==="string"?r:null}function gi(e){let t=v(e.extracted_text_ref)??v(e.extracted_text_uri)??v(e.text_ref);if(t)return t;let r=Q(e.content);return v(r?.extracted_text_ref)??v(r?.extracted_text_uri)??null}function pi(e){let t=v(e.path);return v(e.title)??v(e.name)??(t?di(t):null)}function mi(e){return v(e.hash)??v(e.checksum)??v(e.sha256)??null}function hi(e,t,r){return v(e.revision_id)??v(e.revision)??v(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??v(e.updated_at)??"current"}function ki(e,t){let r={};for(let[n,i]of Object.entries(e)){if(["text","content","content_text","extracted_text","markdown"].includes(n))continue;r[n]=i}return r.source_ref=t.sourceRef,r.source_uri=t.sourceUri,r.status=t.status,r}function yi(e,t){let r=li(e),n=j(r),i=_i(r,n),s=mi(e),c=v(e.status)??"active";return{raw:e,sourceRef:r,sourceUri:i,kind:n.kind,title:pi(e),revision:hi(e,n,s),hash:s,extractedTextUri:gi(e),text:fi(e),metadata:ki(e,{sourceRef:r,sourceUri:i,status:c}),acl:e.permissions??e.acl??{},status:c,updatedAt:v(e.updated_at)??t}}function Ei(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Manifest array parse failed.");return r.map((n)=>{let i=Q(n);if(!i)throw Error("Manifest array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=Q(r);if(!n)throw Error("Manifest object parse failed.");if(Array.isArray(n.items))return n.items.map((i)=>{let s=Q(i);if(!s)throw Error("Manifest items entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=Q(JSON.parse(i));if(!s)throw Error("Manifest JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=Q(JSON.parse(r));if(!n)throw Error("Manifest JSONL entries must be objects.");return n})}async function bi(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 manifest URI: ${e}`);if(r)q(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function vi(e,t,r){if(e.startsWith("s3://"))return bi(e,t,r);if(!ai(e))throw Error(`Manifest not found: ${e}`);return ci(e,"utf8")}function wi(e,t,r){let n=e.replace(/\r\n/g,`
|
|
405
|
+
`);if(!n.trim())return[];let i=[],s=0;while(s<n.length){let c=Math.min(n.length,s+t),u=c;if(c<n.length){let a=n.lastIndexOf(`
|
|
406
406
|
|
|
407
|
-
`,c),d=n.lastIndexOf(". ",c),
|
|
407
|
+
`,c),d=n.lastIndexOf(". ",c),o=Math.max(a,d);if(o>s+Math.floor(t*0.5))u=o+(o===a?2:1)}let _=n.slice(s,u).trim();if(_)i.push({ordinal:i.length,text:_,startOffset:s,endOffset:u});if(u>=n.length)break;s=Math.max(0,u-r)}return i}function Si(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Ti(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t);for(let n of r)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[n.id]);return e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]),r.length}function xi(e,t,r){let n=Ve("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
408
408
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
409
409
|
ON CONFLICT(uri) DO UPDATE SET
|
|
410
410
|
kind = excluded.kind,
|
|
411
411
|
title = excluded.title,
|
|
412
412
|
metadata_json = excluded.metadata_json,
|
|
413
413
|
acl_json = excluded.acl_json,
|
|
414
|
-
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function
|
|
414
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function Ri(e,t,r,n){let i=Ve("rev",`${t}\x00${r.revision}`);e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
415
415
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
416
416
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
417
417
|
hash = excluded.hash,
|
|
418
418
|
extracted_text_uri = excluded.extracted_text_uri,
|
|
419
|
-
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let
|
|
420
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[d,t,"source",a.ordinal,a.text,
|
|
419
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision);if(!s)throw Error(`Failed to upsert source revision: ${r.sourceRef}`);return s.id}function Oi(e,t,r,n,i,s,c){if(!r.text||r.status.toLowerCase()==="deleted")return{chunksInserted:0,redactions:0};let u=ve(r.text,c);if(u.findings.length>0)we(e,{source_uri:r.sourceUri,findings:u.findings,metadata:{source_ref:r.sourceRef,revision:r.revision},created_at:n}),A(e,{event_type:"redaction",action:"source_text_redact",target_uri:r.sourceUri,decision:"redacted",metadata:{findings:u.findings.length,source_ref:r.sourceRef,revision:r.revision},created_at:n});let _=wi(u.text,i,s);for(let a of _){let d=Ve("chk",`${t}\x00${a.ordinal}\x00${a.text}`),o=X({source_ref:r.sourceRef,source_uri:r.sourceUri,source_kind:r.kind,source_revision_id:t,revision:r.revision,hash:r.hash,chunk_id:d,start_offset:a.startOffset,end_offset:a.endOffset,status:r.status,resolver:"open-files-read-only"}),l=yt({source_ref:r.sourceRef,source_uri:r.sourceUri,source_kind:r.kind,source_revision_id:t,revision:r.revision,hash:r.hash,status:r.status,path:v(r.raw.path)??null,mime:v(r.raw.mime)??v(r.raw.content_type)??null,size:ui(r.raw.size)??null},o);e.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
420
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[d,t,"source",a.ordinal,a.text,Si(a.text),a.startOffset,a.endOffset,JSON.stringify(l),n]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[d,a.text,r.title??"",r.sourceUri])}return{chunksInserted:_.length,redactions:u.findings.length}}async function Jt(e){let t=e.now??new Date;if(e.safetyPolicy)z(e.dbPath,e.safetyPolicy);O(e.dbPath);let r=await vi(e.input,e.config,e.safetyPolicy),n=Ei(r);return se({dbPath:e.dbPath,items:n,sourceLabel:e.input,safetyPolicy:e.safetyPolicy,now:t,maxChunkChars:e.maxChunkChars,chunkOverlapChars:e.chunkOverlapChars})}async function se(e){let t=(e.now??new Date).toISOString(),r=e.maxChunkChars??4000,n=e.chunkOverlapChars??200;if(r<500)throw Error("maxChunkChars must be at least 500.");if(n<0||n>=r)throw Error("chunkOverlapChars must be less than maxChunkChars.");if(e.safetyPolicy)z(e.dbPath,e.safetyPolicy);O(e.dbPath);let i=w(e.dbPath);try{return i.transaction(()=>{let c=new Set,u=new Set,_=0,a=0,d=0,o=0;A(i,{event_type:"source_read",action:e.readAction??(e.sourceLabel.startsWith("s3://")?"s3_manifest_read":"local_manifest_read"),target_uri:e.sourceLabel,decision:"allow",metadata:{items:e.items.length,read_only:!0},created_at:t});for(let l of e.items){let f=yi(l,t),g=xi(i,f,t),k=Ri(i,g,f,t);if(c.add(g),u.add(k),f.text||f.status.toLowerCase()==="deleted")a+=Ti(i,k);let h=Oi(i,k,f,t,r,n,e.safetyPolicy);_+=h.chunksInserted,d+=h.redactions}return A(i,{event_type:"write",action:"knowledge_manifest_ingest",target_uri:e.dbPath,decision:"allow",metadata:{items:e.items.length,sources:c.size,revisions:u.size,chunks_inserted:_,redactions:d},created_at:t}),{path:e.sourceLabel,db_path:e.dbPath,items_seen:e.items.length,sources_upserted:c.size,revisions_upserted:u.size,chunks_inserted:_,chunks_deleted:a,redactions:d,skipped:o}})()}finally{i.close()}}import{createHash as Di}from"crypto";import{existsSync as Ui,readFileSync as ji}from"fs";import{basename as Re}from"path";function Te(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function G(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function Yt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Ni(e,t){let r=e.mode;if(typeof r==="string"&&r!=="read_only")throw Error(`Source resolver denied ${t}. Permission mode is ${r}, expected read_only.`);let n=e.denied_purposes;if(Array.isArray(n)&&n.includes(t))throw Error(`Source resolver denied ${t}. Purpose is explicitly denied.`);let i=e.allowed_purposes;if(Array.isArray(i)&&i.length>0&&!i.includes(t))throw Error(`Source resolver denied ${t}. Allowed purposes: ${i.join(", ")}`)}function Ai(e,t,r){if(!t)return r;try{let n=j(e);if(n.kind==="open-files"&&n.entity==="file")return`${e}/revision/${encodeURIComponent(t.revision)}`}catch{return r}return r}function Ii(e,t,r){return e.query(`SELECT id, uri, kind, title, metadata_json, acl_json, updated_at
|
|
421
421
|
FROM sources
|
|
422
422
|
WHERE uri = ? OR uri = ?
|
|
423
423
|
ORDER BY CASE WHEN uri = ? THEN 0 ELSE 1 END
|
|
424
|
-
LIMIT 1`).get(t,r,t)??null}function
|
|
424
|
+
LIMIT 1`).get(t,r,t)??null}function Li(e,t,r){if(r)return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
425
425
|
FROM source_revisions
|
|
426
426
|
WHERE source_id = ? AND revision = ?
|
|
427
427
|
LIMIT 1`).get(t,r)??null;return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
428
428
|
FROM source_revisions
|
|
429
429
|
WHERE source_id = ?
|
|
430
430
|
ORDER BY created_at DESC, revision DESC
|
|
431
|
-
LIMIT 1`).get(t)??null}function
|
|
431
|
+
LIMIT 1`).get(t)??null}function Ci(e,t){if(!t)return 0;return e.query("SELECT COUNT(*) AS n FROM chunks WHERE source_revision_id = ?").get(t)?.n??0}function Pi(e,t,r){if(!t||r<=0)return[];return e.query(`SELECT id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json
|
|
432
432
|
FROM chunks
|
|
433
433
|
WHERE source_revision_id = ?
|
|
434
434
|
ORDER BY ordinal ASC
|
|
435
|
-
LIMIT ?`).all(t,r)}async function
|
|
435
|
+
LIMIT ?`).all(t,r)}async function xe(e){let t=e.purpose??"knowledge_answer",r=Math.max(0,Math.min(e.limit??10,100)),n=(e.now??new Date).toISOString(),i=j(e.sourceRef),s=Ft(e.sourceRef,i),c=Wt(e.sourceRef);if(e.safetyPolicy){if(!e.safetyPolicy.readOnlySourceAccess)throw Error("Safety policy denied source resolution.");z(e.dbPath,e.safetyPolicy)}O(e.dbPath);let u=w(e.dbPath);try{return u.transaction(()=>{let _=Ii(u,s,e.sourceRef);if(!_)return A(u,{event_type:"source_read",action:"open_files_resolve_missing",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:s},created_at:n}),{source_ref:e.sourceRef,source_uri:s,purpose:t,read_only:!0,resolved:!1,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:null,revision:null,content:{mime:null,size:null,hash:null,text_available:!1,chunks_total:0,chunks_returned:0,char_count_returned:0,extracted_text_ref:null,bytes_available:!1,bytes_exposed:!1},chunks:[],citations:[]};let a=Te(_.metadata_json),d=Te(_.acl_json);try{Ni(d,t)}catch(p){throw A(u,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"deny",metadata:{purpose:t,read_only:!0,source_uri:_.uri,error:p instanceof Error?p.message:String(p)},created_at:n}),p}let o=Li(u,_.id,c),l=Te(o?.metadata_json),f=Ci(u,o?.id??null),g=Pi(u,o?.id??null,r),k=Ai(_.uri,o,e.sourceRef),h=g.map((p)=>{let x=Te(p.metadata_json),N={resolver:"open-files-read-only",mode:"local_catalog",purpose:t,read_only:!0,source_ref:G(x,["source_ref"])??k,source_uri:_.uri,source_revision_id:o?.id??null,revision:o?.revision??null,hash:o?.hash??G(x,["hash"]),chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,resolved_at:n},b=X({source_ref:N.source_ref,source_uri:N.source_uri,source_kind:_.kind,source_revision_id:N.source_revision_id,revision:N.revision,hash:N.hash,chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,status:G(x,["status"]),resolver:N.resolver});return{id:p.id,kind:p.kind,ordinal:p.ordinal,text:p.text,token_count:p.token_count,start_offset:p.start_offset,end_offset:p.end_offset,metadata:x,evidence:N,provenance:b}}),m=h.map((p)=>({source_ref:p.evidence.source_ref,source_uri:_.uri,chunk_id:p.id,quote:p.text.slice(0,500),start_offset:p.start_offset,end_offset:p.end_offset,evidence:p.evidence,provenance:p.provenance}));A(u,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:_.uri,revision:o?.revision??null,chunks_returned:h.length,chunks_total:f},created_at:n});let S=G(a,["mime","content_type"])??G(l,["mime","content_type"]),y=Yt(a,["size","size_bytes"])??Yt(l,["size","size_bytes"]);return{source_ref:k,source_uri:_.uri,purpose:t,read_only:!0,resolved:!0,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:{id:_.id,uri:_.uri,kind:_.kind,title:_.title,metadata:a,permissions:d,updated_at:_.updated_at},revision:o?{id:o.id,revision:o.revision,hash:o.hash,extracted_text_uri:o.extracted_text_uri,metadata:l,created_at:o.created_at,reindex_required:l.reindex_required===!0}:null,content:{mime:S,size:y,hash:o?.hash??G(a,["hash","checksum","sha256"]),text_available:f>0,chunks_total:f,chunks_returned:h.length,char_count_returned:h.reduce((p,x)=>p+x.text.length,0),extracted_text_ref:o?.extracted_text_uri??G(l,["extracted_text_ref","extracted_text_uri"]),bytes_available:!1,bytes_exposed:!1},chunks:h,citations:m}})()}finally{u.close()}}function Z(e){return`sha256:${Di("sha256").update(e).digest("hex")}`}function Ki(e){return e.replace(/<script[\s\S]*?<\/script>/gi," ").replace(/<style[\s\S]*?<\/style>/gi," ").replace(/<[^>]+>/g," ").replace(/ /g," ").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/\s+\n/g,`
|
|
436
436
|
`).replace(/\n\s+/g,`
|
|
437
|
-
`).replace(/[ \t]{2,}/g," ").trim()}async function
|
|
437
|
+
`).replace(/[ \t]{2,}/g," ").trim()}async function Mi(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 source URI: ${e}`);if(r)q(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function Fi(e,t){if(t)Y(t);let r=await fetch(e,{headers:{accept:"text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5","user-agent":"@hasna/knowledge source-ingest"}});if(!r.ok)throw Error(`Web source read failed ${r.status}: ${e}`);let n=r.headers.get("content-type"),i=await r.text();return{text:n?.includes("html")?Ki(i):i,mime:n}}function Oe(e){if(e.kind==="file")return Re(e.path);if(e.kind==="s3")return Re(e.key);if(e.kind==="web")return Re(new URL(e.url).pathname)||e.url;return e.path?Re(e.path):e.id}async function Vt(e,t,r){if(e.kind==="file"){if(!Ui(e.path))throw Error(`Source file not found: ${e.path}`);let n=ji(e.path,"utf8");return{text:n,contentSource:"file",title:Oe(e),mime:"text/plain",size:n.length,hash:Z(n),revision:null,extractedTextRef:null,metadata:{path:e.path},permissions:{mode:"read_only"}}}if(e.kind==="s3"){let n=await Mi(e.uri,t,r);return{text:n,contentSource:"s3",title:Oe(e),mime:"text/plain",size:n.length,hash:Z(n),revision:null,extractedTextRef:null,metadata:{bucket:e.bucket,key:e.key},permissions:{mode:"read_only"}}}if(e.kind==="web"){let n=await Fi(e.url,r);return{text:n.text,contentSource:"web",title:Oe(e),mime:n.mime,size:n.text.length,hash:Z(n.text),revision:null,extractedTextRef:null,metadata:{url:e.url},permissions:{mode:"read_only"}}}throw Error(`Direct source reading is not available for ${e.uri}`)}async function Wi(e,t,r){if(e.startsWith("open-files://"))throw Error("Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.");let n=j(e);return{text:(await Vt(n,t,r)).text,contentSource:"extracted_text_ref"}}async function $i(e){let t=await xe({dbPath:e.dbPath,sourceRef:e.sourceRef,purpose:e.purpose??"knowledge_index",limit:100,safetyPolicy:e.safetyPolicy,now:e.now});if(!t.resolved)throw Error("Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.");if(t.revision?.extracted_text_uri&&!t.content.text_available){let n=await Wi(t.revision.extracted_text_uri,e.config,e.safetyPolicy);return{text:n.text,contentSource:n.contentSource,title:t.source?.title??null,mime:t.content.mime,size:n.text.length,hash:t.revision.hash??Z(n.text),revision:t.revision.revision,extractedTextRef:t.revision.extracted_text_uri,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}if(t.chunks.length===0)throw Error("Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.");let r=t.chunks.map((n)=>n.text).join(`
|
|
438
438
|
|
|
439
|
-
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??
|
|
439
|
+
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??Z(r),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function Xi(e,t,r,n){let i=r.hash??Z(r.text),s={...r.metadata,source_ref:e,content_source:r.contentSource,read_only:!0},c={source_ref:e,name:r.title??Oe(t),mime:r.mime??"text/plain",size:r.size??r.text.length,hash:i,revision:r.revision??i,status:"active",updated_at:new Date().toISOString(),permissions:{mode:"read_only",allowed_purposes:[n],...r.permissions},metadata:s,extracted_text_ref:r.extractedTextRef,extracted_text:r.text};if(t.kind==="open-files"){if(t.entity==="file")c.file_id=t.id;if(t.entity==="source")c.source_id=t.id,c.path=t.path}if(t.kind==="file")c.path=t.path;if(t.kind==="s3")c.path=t.key;if(t.kind==="web")c.url=t.url;return c}async function Qt(e){let t=e.purpose??"knowledge_index",r=j(e.sourceRef),n=r.kind==="open-files"?await $i(e):await Vt(r,e.config,e.safetyPolicy),i=Xi(e.sourceRef,r,n,t);return{...await se({dbPath:e.dbPath,items:[i],sourceLabel:e.sourceRef,readAction:"source_ref_ingest_read",safetyPolicy:e.safetyPolicy,now:e.now}),source_ref:e.sourceRef,content_source:n.contentSource,read_only:!0,hash:String(i.hash)}}import{createHash as qi,randomUUID as Zt}from"crypto";function Hi(e){return`sha256:${qi("sha256").update(e).digest("hex")}`}function er(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function tr(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:{}}function H(e){return typeof e==="string"&&e.length>0?e:null}function Bi(e){let t=tr(e),r=H(t.url)??H(t.uri)??H(t.sourceUrl);if(!r)return null;return{url:r,title:H(t.title)??H(t.name),snippet:H(t.snippet)??H(t.text)??H(t.description),provider_metadata:t}}function Ne(e,t){if(Array.isArray(e)){for(let i of e)Ne(i,t);return}let r=Bi(e);if(r)t.set(r.url,r);let n=tr(e);for(let i of["sources","results","citations","annotations","output"])if(n[i])Ne(n[i],t)}function zi(e,t){return Array.from({length:Math.min(t,3)},(r,n)=>({url:`https://example.com/knowledge-web-${n+1}`,title:`Fake web source ${n+1}`,snippet:`Deterministic web-search fixture for "${e}"`,provider_metadata:{fake:!0,rank:n+1}}))}async function Gi(e){let{generateText:t}=await import("ai"),{createOpenAI:r}=await import("@ai-sdk/openai"),n=W(e.config,"openai"),i=r({apiKey:e.env[n.api_key_env],baseURL:n.base_url}),s=i.tools?.webSearch;if(!s)throw Error("OpenAI provider does not expose tools.webSearch.");return t({model:i(e.model),prompt:e.query,tools:{web_search:s({externalWebAccess:!0,searchContextSize:"medium",...e.domains.length>0?{allowedDomains:e.domains}:{}})},toolChoice:{type:"tool",toolName:"web_search"}})}async function Ji(e){let{generateText:t}=await import("ai"),{createAnthropic:r}=await import("@ai-sdk/anthropic"),n=W(e.config,"anthropic"),i=r({apiKey:e.env[n.api_key_env],baseURL:n.base_url}),s=i.tools?.webSearch_20250305??i.tools?.webSearch;if(!s)throw Error("Anthropic provider does not expose a web search tool.");return t({model:i(e.model),prompt:e.query,tools:{web_search:s({maxUses:e.maxUses,...e.domains.length>0?{allowedDomains:e.domains}:{}})}})}async function Yi(e,t,r){if(!e.fileResults||t.length===0)return 0;let n=t.map((s)=>{let c=[s.title,s.snippet,s.url].filter(Boolean).join(`
|
|
440
|
+
`),u=Hi(c);return{source_ref:s.url,name:s.title??s.url,url:s.url,mime:"text/plain",hash:u,revision:u,status:"active",updated_at:r,permissions:{mode:"read_only",allowed_purposes:["knowledge_answer","knowledge_index"]},metadata:{source_ref:s.url,content_source:"provider_web_search",provider_metadata:s.provider_metadata},extracted_text:c}});return(await se({dbPath:e.dbPath,items:n,sourceLabel:`web-search:${e.query}`,readAction:"provider_web_search_file_results",safetyPolicy:e.safetyPolicy,now:new Date(r)})).sources_upserted}async function rr(e){let t=e.query.trim();if(!t)throw Error("Web search query is required.");let r=e.env??process.env,n=(e.now??new Date).toISOString(),i=Math.max(1,Math.min(e.limit??5,20)),s=Math.max(1,Math.min(e.maxUses??3,10)),c=e.domains??[],u=$(e.modelRef??(e.provider?`${e.provider}:${W(e.config,e.provider).default_model}`:"default"),e.config),_=U(u),a=e.provider??_.provider,d=_.provider===a?_.model:W(e.config,a).default_model,o=`run_${Zt()}`;if(!e.fake&&e.safetyPolicy)Y(e.safetyPolicy);if(!e.fake&&a!=="openai"&&a!=="anthropic")throw Error(`Provider ${a} does not expose native web search yet.`);if(!e.fake)B(a,e.config,r);O(e.dbPath);let l=w(e.dbPath);try{l.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
441
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[o,"provider-web-search",t,"running",a,d,JSON.stringify({domains:c,max_uses:s,fake:e.fake===!0}),n,n]),A(l,{event_type:"source_read",action:e.fake?"fake_provider_web_search":"provider_web_search",target_uri:t,decision:"allow",metadata:{provider:a,model:d,domains:c,max_uses:s},created_at:n})}finally{l.close()}let f="",g=[],k={input_tokens:er(t),output_tokens:0,cost_usd:0},h=[];if(e.fake)g=zi(t,i),f=`Fake web search answer for: ${t}`,k.output_tokens=er(f);else{let y=a==="openai"?await Gi({query:t,model:d,config:e.config,env:r,maxUses:s,domains:c}):await Ji({query:t,model:d,config:e.config,env:r,maxUses:s,domains:c});f=y.text;let p=new Map;Ne(y.sources,p),Ne(y.toolResults,p),g=Array.from(p.values()).slice(0,i);let x=_e({provider:a,model:d,usage:y.usage,providerMetadata:y.providerMetadata});k={input_tokens:x.input_tokens,output_tokens:x.output_tokens,cost_usd:x.cost_usd}}let m=await Yi(e,g,n),S=w(e.dbPath);try{S.run("UPDATE runs SET status = ?, metadata_json = ?, updated_at = ? WHERE id = ?",["completed",JSON.stringify({domains:c,max_uses:s,sources:g.length,filed_sources:m,fake:e.fake===!0}),n,o]),S.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
442
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${Zt()}`,o,"info","provider_web_search_completed",JSON.stringify({sources:g.length,filed_sources:m}),n]),fe(S,{run_id:o,provider:a,model:d,input_tokens:k.input_tokens,output_tokens:k.output_tokens,cost_usd:k.cost_usd,metadata:{web_search:!0,sources:g.length,filed_sources:m},created_at:n})}finally{S.close()}if(g.length===0)h.push("no_web_sources_returned");return{run_id:o,query:t,provider:a,model:d,answer:f,sources:g,filed_sources:m,usage:k,warnings:h}}import{createHash as Vi,randomUUID as Qi}from"crypto";var nr=[{kind:"schema",prefix:"schemas/",description:"Machine-readable agent schemas and source rules."},{kind:"index",prefix:"indexes/",description:"Small orientation indexes and future shard manifests."},{kind:"log",prefix:"logs/",description:"Append-only JSONL run and wiki-maintenance log partitions."},{kind:"run",prefix:"runs/",description:"Prompt/tool/cost ledgers and generated output records."},{kind:"wiki_page",prefix:"wiki/",description:"Generated cited Markdown pages, not raw source files."},{kind:"export",prefix:"exports/",description:"Portable exports and snapshots of derived knowledge state."}];function ir(e){let t=typeof e==="string"?Buffer.from(e):Buffer.from(e);return{hash:`sha256:${Vi("sha256").update(t).digest("hex")}`,size_bytes:t.byteLength}}function sr(e){return nr.find((r)=>e.startsWith(r.prefix))?.kind??"artifact"}function or(e,t,r="global"){let n=Qe(e,t),i=e.storage.s3??null,s=i?.prefix?.replace(/^\/+|\/+$/g,"")??"",c=i?`s3://${i.bucket}/${s?`${s}/`:""}`:"";return{scope:r,mode:e.mode,storage_type:e.storage.type,workspace_home:t.home,local_layout:{app_path:te,config_path:t.configPath,json_store_path:t.jsonStorePath,knowledge_db_path:t.knowledgeDbPath,directories:{artifacts:t.artifactsDir,cache:t.cacheDir,exports:t.exportsDir,indexes:t.indexesDir,logs:t.logsDir,runs:t.runsDir,schemas:t.schemasDir,wiki:t.wikiDir}},artifact_store:{type:e.storage.type,artifacts_root:e.storage.artifacts_root,uri_prefix:e.storage.type==="s3"?c:`file://${t.artifactsDir}/`,s3:i?{bucket:i.bucket,prefix:s,region:i.region??null,profile:i.profile??null,server_side_encryption:i.server_side_encryption??null,kms_key_configured:Boolean(i.kms_key_id)}:null},source_ownership:{owner:"open-files",preferred_ref:e.sources.preferred_ref,allowed_schemes:e.sources.allowed_schemes,raw_source_bytes_stored_in_open_knowledge:!1,stores:["source refs","source revisions and hashes","citation spans","redacted extracted chunks","embeddings","generated wiki artifacts","indexes","run ledgers"],does_not_store:["raw open-files bytes","S3 object credentials","connector secrets","hosted tenant ownership state"]},generated_artifacts:nr,scalability:{catalog:"knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.",indexes:"Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.",logs:"Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.",markdown:"Markdown pages are the readable wiki layer over DB/object-store state."},warnings:n.warnings}}function Qe(e,t){let r=[],n=[];if(!t.home.endsWith(te))n.push(`Workspace home does not end with ${te}: ${t.home}`);if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)r.push("storage.s3.bucket is required when storage.type is s3.");if(!e.storage.s3?.prefix)n.push("storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.");if(e.mode==="local")n.push("storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.")}if(e.storage.type==="local"&&e.storage.s3)n.push("storage.s3 is configured but ignored while storage.type is local.");if(e.sources.preferred_ref!=="open-files")n.push("sources.preferred_ref should stay open-files for durable company knowledge.");if(!e.sources.allowed_schemes.includes("open-files"))r.push("sources.allowed_schemes must include open-files.");return{ok:r.length===0,errors:r,warnings:n}}function ar(e,t,r=new Date){let n=r.toISOString(),i=e.prepare(`
|
|
440
443
|
INSERT INTO storage_objects (
|
|
441
444
|
id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
|
|
442
445
|
)
|
|
@@ -448,7 +451,7 @@ ${l}`;else{let{generateText:p}=await import("ai"),S=await gt(i,{config:e.config,
|
|
|
448
451
|
size_bytes = excluded.size_bytes,
|
|
449
452
|
metadata_json = excluded.metadata_json,
|
|
450
453
|
updated_at = excluded.updated_at
|
|
451
|
-
`);e.transaction((c)=>{for(let u of c)i.run(
|
|
454
|
+
`);e.transaction((c)=>{for(let u of c)i.run(Qi(),u.uri,u.kind,u.content_type??null,u.hash??null,u.size_bytes??null,JSON.stringify({key:u.key,...u.metadata??{}}),n,n)})(t)}import{createHash as Zi}from"crypto";function es(e){let t=String(e.getUTCFullYear()),r=String(e.getUTCMonth()+1).padStart(2,"0"),n=String(e.getUTCDate()).padStart(2,"0");return{year:t,month:r,day:n}}function Ze(e,t){return`${e}_${Zi("sha256").update(t).digest("hex").slice(0,20)}`}function ts(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function rs(){return`# Knowledge Agent Schema v1
|
|
452
455
|
|
|
453
456
|
## Source Rules
|
|
454
457
|
|
|
@@ -473,7 +476,7 @@ ${l}`;else{let{generateText:p}=await import("ai"),S=await gt(i,{config:e.config,
|
|
|
473
476
|
## Lint Rules
|
|
474
477
|
|
|
475
478
|
- Flag stale pages, missing citations, contradictions, orphan pages, duplicate pages, and unresolved source refs.
|
|
476
|
-
`}function
|
|
479
|
+
`}function ns(){return`# Knowledge Index
|
|
477
480
|
|
|
478
481
|
This is a compact orientation index for agents. It is not the full search index.
|
|
479
482
|
|
|
@@ -488,19 +491,19 @@ This is a compact orientation index for agents. It is not the full search index.
|
|
|
488
491
|
|
|
489
492
|
Raw source files are resolved through open-files. This app stores source refs,
|
|
490
493
|
citations, chunks, generated wiki artifacts, indexes, and run records.
|
|
491
|
-
`}function
|
|
494
|
+
`}function cr(){return`# Wiki
|
|
492
495
|
|
|
493
496
|
Generated durable knowledge pages live here.
|
|
494
497
|
|
|
495
498
|
Pages should be concise, cited, and organized for both humans and agents.
|
|
496
|
-
`}async function
|
|
497
|
-
`,content_type:"application/x-ndjson"}],
|
|
498
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[u,t,"wiki",0,i,
|
|
499
|
+
`}async function dr(e,t=new Date){let{year:r,month:n,day:i}=es(t),s="schemas/v1.md",c="indexes/root.md",u="wiki/README.md",_=`logs/${r}/${n}/${i}.jsonl`,a={ts:t.toISOString(),event:"wiki_layout_initialized",schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md"},d=[{key:"schemas/v1.md",body:rs(),content_type:"text/markdown"},{key:"indexes/root.md",body:ns(),content_type:"text/markdown"},{key:"wiki/README.md",body:cr(),content_type:"text/markdown"},{key:_,body:`${JSON.stringify(a)}
|
|
500
|
+
`,content_type:"application/x-ndjson"}],o=await Promise.all(d.map(async(l)=>{let f=await e.put(l);return{key:f.key,uri:f.uri,kind:sr(l.key),content_type:l.content_type,metadata:{provenance:$e({generated_from:"wiki_layout_init",artifact_key:l.key,citation_required:l.key.startsWith("wiki/")||l.key.startsWith("indexes/")})},...ir(l.body)}}));return{schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md",log_key:_,artifacts:o,written:["schemas/v1.md","indexes/root.md","wiki/README.md",_]}}function et(e){let t=e.metadata?.provenance;if(t&&typeof t==="object"&&!Array.isArray(t))return t;return $e({generated_from:"wiki_layout_init",artifact_key:e.key})}function is(e,t,r,n,i,s){let c=et(n),u=Ze("chk",`${t}\x00${n.hash??n.uri}`),_=e.query("SELECT id FROM chunks WHERE wiki_page_id = ?").all(t);for(let a of _)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[a.id]);e.run("DELETE FROM chunks WHERE wiki_page_id = ?",[t]),e.run(`INSERT INTO chunks (id, wiki_page_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
501
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[u,t,"wiki",0,i,ts(i),0,i.length,JSON.stringify({artifact_key:n.key,artifact_uri:n.uri,content_hash:n.hash??null,provenance:c}),s]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[u,i,r,n.uri])}function ur(e,t,r=new Date){let n=r.toISOString(),i=t.find((c)=>c.key.endsWith("indexes/root.md")),s=t.find((c)=>c.key.endsWith("wiki/README.md"));if(i)e.run(`INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
|
|
499
502
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
500
503
|
ON CONFLICT(kind, name, shard_key) DO UPDATE SET
|
|
501
504
|
artifact_uri = excluded.artifact_uri,
|
|
502
505
|
metadata_json = excluded.metadata_json,
|
|
503
|
-
updated_at = excluded.updated_at`,[
|
|
506
|
+
updated_at = excluded.updated_at`,[Ze("idx","root:indexes/root.md"),"root","root",i.uri,"root",JSON.stringify({artifact_key:i.key,content_hash:i.hash??null,provenance:et(i)}),n,n]);if(s){let c=Ze("wiki","wiki/README.md");e.run(`INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
|
|
504
507
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
505
508
|
ON CONFLICT(path) DO UPDATE SET
|
|
506
509
|
title = excluded.title,
|
|
@@ -508,7 +511,7 @@ Pages should be concise, cited, and organized for both humans and agents.
|
|
|
508
511
|
content_hash = excluded.content_hash,
|
|
509
512
|
status = excluded.status,
|
|
510
513
|
metadata_json = excluded.metadata_json,
|
|
511
|
-
updated_at = excluded.updated_at`,[c,"wiki/README.md","Wiki",
|
|
514
|
+
updated_at = excluded.updated_at`,[c,"wiki/README.md","Wiki",s.uri,s.hash??null,"active",JSON.stringify({artifact_key:s.key,provenance:et(s)}),n,n]),is(e,c,"Wiki",s,cr(),n)}}class lr{options;ensuredWorkspace;cachedConfig;constructor(e={}){this.options=e}get scope(){return this.options.scope??"global"}get workspace(){return this.ensuredWorkspace??nt(this.options.scope,this.options.cwd)}ensureWorkspace(){if(!this.ensuredWorkspace)this.ensuredWorkspace=rt(this.workspace.home);return this.ensuredWorkspace}jsonStorePath(){return this.ensureWorkspace().jsonStorePath}config(){if(!this.cachedConfig){let e=this.ensureWorkspace();this.cachedConfig=it(e.configPath)}return this.cachedConfig}safetyPolicy(){return qt(this.config(),this.ensureWorkspace())}artifactStore(){return _t(this.config(),this.ensureWorkspace())}storageContract(){return or(this.config(),this.ensureWorkspace(),this.scope)}validateStorage(){return Qe(this.config(),this.ensureWorkspace())}paths(){let e=this.ensureWorkspace();return{ok:!0,scope:this.scope,home:e.home,config_path:e.configPath,json_store_path:e.jsonStorePath,knowledge_db_path:e.knowledgeDbPath,artifacts_dir:e.artifactsDir,indexes_dir:e.indexesDir,logs_dir:e.logsDir,runs_dir:e.runsDir,schemas_dir:e.schemasDir,wiki_dir:e.wikiDir,config:this.config(),message:e.home}}initDb(){return O(this.ensureWorkspace().knowledgeDbPath)}dbStats(){let e=this.ensureWorkspace();return O(e.knowledgeDbPath),ct(e.knowledgeDbPath)}async initWiki(){let e=this.ensureWorkspace();O(e.knowledgeDbPath);let t=await dr(this.artifactStore()),r=w(e.knowledgeDbPath);try{ar(r,t.artifacts),ur(r,t.artifacts)}finally{r.close()}return t}async ingestManifest(e){let t=this.ensureWorkspace();return Jt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}async ingestSource(e,t){let r=this.ensureWorkspace();return Qt({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t,config:this.config(),safetyPolicy:this.safetyPolicy()})}async resolveSource(e,t={}){let r=this.ensureWorkspace();return xe({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t.purpose,limit:t.limit,safetyPolicy:this.safetyPolicy()})}async consumeOutbox(e){let t=this.ensureWorkspace();return Gt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}providerStatus(e=process.env){return ht(this.config(),e)}modelRegistry(){return Fe(this.config())}embeddingStatus(){let e=this.ensureWorkspace();return xt(e.knowledgeDbPath)}async indexEmbeddings(e={}){let t=this.ensureWorkspace();return Tt({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async semanticSearch(e){let t=this.ensureWorkspace();return pe({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async search(e){let t=this.ensureWorkspace();return ye({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async retrieveContext(e){let t=this.ensureWorkspace();return be({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async runPrompt(e){let t=this.ensureWorkspace();return Kt({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async webSearch(e){let t=this.ensureWorkspace();return rr({...e,dbPath:t.knowledgeDbPath,config:this.config(),safetyPolicy:this.safetyPolicy()})}}function _r(e={}){return new lr(e)}import{basename as os}from"path";var oe={name:"@hasna/knowledge",version:"0.2.18",description:"Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",type:"module",bin:{knowledge:"bin/open-knowledge.js","open-knowledge":"bin/open-knowledge.js","open-knowledge-mcp":"bin/open-knowledge-mcp.js"},files:["bin","src","docs","LICENSE","README.md"],scripts:{test:"bun test","test:cli":"bun test tests/cli.test.ts",build:"bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",prepublishOnly:"bun run build",postinstall:"bun run build"},keywords:["knowledge","cli","agents","json","notes","local","store"],license:"Apache-2.0",publishConfig:{registry:"https://registry.npmjs.org",access:"public"},repository:{type:"git",url:"git+https://github.com/hasna/knowledge.git"},bugs:{url:"https://github.com/hasna/knowledge/issues"},author:"Hasna Inc. <hasna@example.com>",engines:{bun:">=1.0",node:">=18"},dependencies:{"@aws-sdk/client-s3":"^3.1063.0","@aws-sdk/credential-providers":"^3.1063.0","@ai-sdk/anthropic":"^3.0.81","@ai-sdk/deepseek":"^2.0.35","@ai-sdk/openai":"^3.0.68","@modelcontextprotocol/sdk":"^1.29.0",ai:"^6.0.197",zod:"^4.3.6"},devDependencies:{"@types/bun":"^1.3.14"}};var fr={debug:0,info:1,warn:2,error:3},as=()=>{if(process.env.DEBUG)return"debug";if(process.env.LOG_LEVEL==="debug")return"debug";if(process.env.LOG_LEVEL==="warn")return"warn";if(process.env.LOG_LEVEL==="error")return"error";return"info"};function J(e,t,r){if(fr[e]<fr[as()])return;let n={debug:"[DEBUG]",info:"[INFO]",warn:"[WARN]",error:"[ERROR]"}[e],i=r?`${n} ${t} ${JSON.stringify(r)}`:`${n} ${t}`;if(e==="error")console.error(i);else console.error(i)}var gr=["add","list","get","delete","update","archive","restore","upsert","untag","export","prune","dedupe","stats","paths","storage","db","wiki","source","ingest","reindex","search","web","ask","build","embeddings","providers","safety","help"],pr={ls:"list",rm:"delete",edit:"update",unarchive:"restore",knowledge:"ask"};function cs(e){let t=[],r={};for(let n=0;n<e.length;n+=1){let i=e[n];if(!i.startsWith("-")){t.push(i);continue}switch(i){case"--json":r.json=!0;break;case"--yes":case"-y":r.yes=!0;break;case"--help":case"-h":r.help=!0;break;case"--version":case"-v":r.version=!0;break;case"--desc":r.desc=!0;break;case"--page":case"-p":r.page=Number(e[n+1]),n+=1;break;case"--limit":case"-l":r.limit=Number(e[n+1]),n+=1;break;case"--search":case"-s":r.search=e[n+1],n+=1;break;case"--sort":r.sort=e[n+1],n+=1;break;case"--id":r.id=e[n+1],n+=1;break;case"--store":r.store=e[n+1],n+=1;break;case"--title":r.title=e[n+1],n+=1;break;case"--content":r.content=e[n+1],n+=1;break;case"--url":r.url=e[n+1],n+=1;break;case"--tag":case"-t":r.tag=e[n+1],n+=1;break;case"--format":r.format=e[n+1],n+=1;break;case"--completions":r.completions=e[n+1],n+=1;break;case"--purpose":r.purpose=e[n+1],n+=1;break;case"--model":r.model=e[n+1],n+=1;break;case"--dimensions":r.dimensions=Number(e[n+1]),n+=1;break;case"--semantic":r.semantic=!0;break;case"--context":r.context=!0;break;case"--generate":r.generate=!0;break;case"--approve-write":r.approveWrite=!0;break;case"--provider":r.provider=e[n+1],n+=1;break;case"--domain":r.domain=[...r.domain??[],e[n+1]],n+=1;break;case"--file-results":r.fileResults=!0;break;case"--fake":r.fake=!0;break;case"--no-color":r.noColor=!0;break;case"--scope":r.scope=e[n+1],n+=1;break;case"--older-than":r.olderThan=Number(e[n+1]),n+=1;break;case"--empty":r.empty=!0;break;case"--archived":r.archived=!0;break;case"--include-archived":r.includeArchived=!0;break;default:throw Error(`Unknown flag: ${i}. Run 'open-knowledge --help' for valid options.`)}}return{positional:t,flags:r}}function ds(e){if(!e)return"";return pr[e]??e}function us(e,t){let r=Array.from({length:e.length+1},()=>Array(t.length+1).fill(0));for(let n=0;n<=e.length;n+=1)r[n][0]=n;for(let n=0;n<=t.length;n+=1)r[0][n]=n;for(let n=1;n<=e.length;n+=1)for(let i=1;i<=t.length;i+=1){let s=e[n-1]===t[i-1]?0:1;r[n][i]=Math.min(r[n-1][i]+1,r[n][i-1]+1,r[n-1][i-1]+s)}return r[e.length][t.length]}function ls(e){if(!e)return"";let t=[...gr,...Object.keys(pr)],r="",n=Number.POSITIVE_INFINITY;for(let i of t){let s=us(e,i);if(s<n)n=s,r=i}return n<=3?r:""}function _s(){return os(process.argv[1]??"")==="knowledge"}function fs(){console.log(`open-knowledge - local agent knowledge store
|
|
512
515
|
|
|
513
516
|
Usage:
|
|
514
517
|
open-knowledge <command> [options]
|
|
@@ -536,6 +539,7 @@ Commands:
|
|
|
536
539
|
ingest source <source-ref> Ingest a read-only source ref into knowledge.db
|
|
537
540
|
reindex outbox <file|s3://> Consume open-files change events and invalidate chunks
|
|
538
541
|
search <query> Hybrid search sources, wiki pages, indexes, or context
|
|
542
|
+
web search <query> Provider-native web search with citations
|
|
539
543
|
ask|build <prompt> Build a read-only citation answer/context pack
|
|
540
544
|
embeddings status|index|search Build/query local vector embeddings
|
|
541
545
|
providers status|models|check Inspect AI SDK provider config and credentials
|
|
@@ -552,6 +556,9 @@ Global Options:
|
|
|
552
556
|
--context Return a reranked citation context pack for search
|
|
553
557
|
--generate Call AI SDK text generation for ask/build
|
|
554
558
|
--approve-write Record approval intent for future durable wiki writes
|
|
559
|
+
--provider <name> Provider override for web search
|
|
560
|
+
--domain <domain> Restrict provider web search to a domain
|
|
561
|
+
--file-results File web snippets as web source refs
|
|
555
562
|
--fake Use deterministic fake embeddings for local tests
|
|
556
563
|
--scope local|global|project Store scope (default: global ~/.hasna/apps/knowledge/)
|
|
557
564
|
--no-color Disable color output
|
|
@@ -589,5 +596,5 @@ Export Options:
|
|
|
589
596
|
|
|
590
597
|
Prune Options:
|
|
591
598
|
--older-than <days> Remove items older than N days
|
|
592
|
-
--empty Remove items with empty content`)}function
|
|
593
|
-
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=Yi(t[0]),i=1;if(Zi()&&n&&!cr.includes(n))n="ask",i=0;if(!n||r.help||n==="help"){ts(t[1]);return}let o=or({scope:r.scope}),c=r.store;if(!c)if(r.scope==="project"||r.scope==="local")c=o.jsonStorePath();else c=Ne();if(n==="paths"){E(o.paths(),r.json);return}if(n==="storage"){let a=t[1]??"status";if(a==="status"){let d=o.storageContract(),s=o.validateStorage();E({ok:s.ok,...d,validation:s,message:`${d.storage_type} artifact storage at ${d.artifact_store.uri_prefix}`},r.json);return}if(a==="validate"){let d=o.validateStorage();E({ok:d.ok,validation:d,message:d.ok?"Storage contract valid":`Storage contract invalid: ${d.errors.join("; ")}`},r.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(n==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let s=o.initDb();E({ok:!0,...s,message:`Initialized ${s.path}`},r.json);return}let d=o.dbStats();E({ok:!0,path:o.workspace.knowledgeDbPath,...d,message:`knowledge.db schema v${d.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let d=await o.initWiki();E({ok:!0,...d,message:`Initialized wiki layout in ${o.workspace.home}`},r.json);return}if(n==="safety"){let a=t[1]??"status",d=o.ensureWorkspace(),s=o.safetyPolicy();o.initDb();let l=w(d.knowledgeDbPath);try{if(a==="status"){E({ok:!0,mode:s.mode,workspace:d.home,allow_write_roots:s.allowWriteRoots,read_only_source_access:s.readOnlySourceAccess,network:s.network,redaction:s.redaction,approvals:s.approvals,message:`Safety policy: ${s.mode}`},r.json);return}if(a==="check"){let f=t[2]??"generated_write",g=t[3]??null,k;try{if(f==="web_search")me(s),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!g)throw Error("safety check s3_read requires an s3:// target.");X(g,s),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"}}else k=qt(l,s,f,g);A(l,{event_type:"safety_check",action:f,target_uri:g,decision:k.decision==="allow"?"allow":"requires_approval",metadata:k}),E({ok:!0,...k,message:`Safety check ${k.decision}`},r.json);return}catch(h){throw A(l,{event_type:"safety_check",action:f,target_uri:g,decision:"deny",metadata:{error:h instanceof Error?h.message:String(h)}}),h}}if(a==="approve"){let f=t[2]??"generated_write",g=t[3]??null,k=$t(l,{action:f,target_uri:g,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});A(l,{event_type:"approval",action:f,target_uri:g,decision:"allow",metadata:{approval_id:k.id}}),E({ok:!0,...k,action:f,target_uri:g,message:`Approved ${f}`},r.json);return}if(a==="audit"){let f=l.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((g)=>({id:g.id,event_type:g.event_type,action:g.action,target_uri:g.target_uri,decision:g.decision,metadata:JSON.parse(g.metadata_json),created_at:g.created_at}));E({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(a==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let g=ke(f,s);if(g.findings.length>0)Ee(l,{source_uri:"safety://redact",findings:g.findings,metadata:{command:"safety redact"}});A(l,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:g.findings.length>0?"redacted":"allow",metadata:{findings:g.findings.length}}),E({ok:!0,text:g.text,findings:g.findings,message:`Redacted ${g.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{l.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge source resolve <source-ref>");let s=await o.resolveSource(d,{purpose:r.purpose,limit:r.limit});E({ok:!0,...s,message:s.resolved?`Resolved ${s.source_ref} (${s.content.chunks_returned}/${s.content.chunks_total} chunks)`:`Source not indexed: ${d}`},r.json);return}if(n==="ingest"){let a=t[1]??"";if(a==="manifest"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let s=await o.ingestManifest(d);E({ok:!0,...s,message:`Ingested ${s.items_seen} manifest item(s)`},r.json);return}if(a==="source"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest source <source-ref>");let s=await o.ingestSource(d,r.purpose);E({ok:!0,...s,message:`Ingested source ${s.source_ref} (${s.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let s=await o.consumeOutbox(d);E({ok:!0,...s,message:`Consumed ${s.events_seen} outbox event(s)`},r.json);return}if(n==="embeddings"){let a=t[1]??"status";if(a==="status"){let d=o.embeddingStatus();E({ok:!0,...d,message:`${d.total_vector_entries} vector index entries`},r.json);return}if(a==="index"){let d=await o.indexEmbeddings({limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`Embedded ${d.chunks_embedded} chunk(s)`},r.json);return}if(a==="search"){let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge embeddings search <query>");let s=await o.semanticSearch({query:d,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...s,message:`${s.results.length} semantic result(s)`},r.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(n==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");if(r.context){let s=await o.retrieveContext({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...s,message:`${s.excerpts.length} context excerpt(s)`},r.json);return}let d=await o.search({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`${d.results.length} search result(s)`},r.json);return}if(n==="ask"||n==="build"){let a=t.slice(i).join(" ");if(!a)throw Error("Usage: open-knowledge ask <prompt>");let d=await o.runPrompt({prompt:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake,generate:r.generate,approveWrite:r.approveWrite});E({ok:!0,...d,message:d.generated?"Generated answer with citations":"Prepared citation context draft"},r.json);return}if(n==="providers"){let a=t[1]??"status";if(a==="status"){let d=o.providerStatus(),s=d.providers.filter((l)=>l.configured).length;E({ok:!0,...d,message:`${s}/${d.providers.length} provider credential(s) configured`},r.json);return}if(a==="models"){let d=o.modelRegistry();E({ok:!0,models:d,message:`${d.length} model alias(es)`},r.json);return}if(a==="check"){let d=t[2]??"default",s=B(d,o.config()),l=M(s),f=ee(l.provider,o.config());E({ok:!0,target:d,model_ref:s,provider:l.provider,model:l.model,credential:f,message:`${l.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(Ae(c),n==="add"){let a=t[1],d=t[2];if(!a||!d)throw Error("Usage: open-knowledge add <title> <content>");D(c,()=>{let s=C(c),l={id:Ie(),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};s.items.push(l),K(c,s),H("info","Item added",{id:l.id,title:l.title}),E({ok:!0,item:l,message:`Added ${l.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");D(c,()=>{let a=C(c),d=Number.isFinite(r.page)&&r.page>0?r.page:1,s=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,l=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",g=r.format==="table"||!r.json&&!r.format&&rs(r),k=r.json||r.format==="json",h=a.items;if(r.archived)h=h.filter((b)=>b.archived===!0);else if(!r.includeArchived)h=h.filter((b)=>!b.archived);if(l)h=h.filter((b)=>b.title.toLowerCase().includes(l)||b.content.toLowerCase().includes(l));if(f)h=h.filter((b)=>b.tags&&b.tags.map((xe)=>xe.toLowerCase()).includes(f));let{sorted:m,sort:x,direction:y}=ns(h,r),p=(d-1)*s,S=m.slice(p,p+s),R=Math.max(1,Math.ceil(m.length/s));if(k){E({ok:!0,page:d,limit:s,total:m.length,total_pages:R,sort:x,direction:y,items:S},!0);return}if(S.length===0){E(`No items found (search=${l||"none"}, tag=${f||"none"})`,!1);return}if(g){let b=(F)=>F,xe=`${b("ID")} ${b("TITLE")} ${b("CREATED")} ${b("URL")} ${b("TAGS")}`;console.log(xe);for(let F of S)console.log(`${F.id} ${b(F.title)} ${F.created_at} ${F.url?b(F.url):""} ${F.tags?.length?b(`[${F.tags.join(", ")}]`):""}`);console.log(`Page ${d}/${R} | showing ${S.length} of ${m.length} | sort=${x} ${y} | search=${l||"none"} | tag=${f||"none"}`)}else{for(let b of S)console.log(`${b.id} ${b.title} ${b.created_at}${b.url?` ${b.url}`:""}${b.tags?.length?` [${b.tags.join(", ")}]`:""}`);console.log(`Page ${d}/${R} | showing ${S.length} of ${m.length} | sort=${x} ${y} | search=${l||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){ne(r),D(c,()=>{let d=C(c).items.find((s)=>s.id===r.id||s.short_id===r.id);if(!d)throw Error(`Item not found: ${r.id}`);E({ok:!0,item:d,message:`${d.id}: ${d.title}`},r.json)});return}if(n==="update"){ne(r),D(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let s=a.items[d];if(r.title!==void 0)s.title=r.title;if(r.content!==void 0)s.content=r.content;if(r.url!==void 0)s.url=r.url;if(r.tag!==void 0){if(s.tags=s.tags||[],!s.tags.map((l)=>l.toLowerCase()).includes(r.tag.toLowerCase()))s.tags.push(r.tag)}s.updated_at=new Date().toISOString(),a.items[d]=s,K(c,a),E({ok:!0,item:s,message:`Updated ${s.id}`},r.json)});return}if(n==="archive"||n==="restore"){ne(r),D(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let s=a.items[d];s.archived=n==="archive",s.updated_at=new Date().toISOString(),a.items[d]=s,K(c,a),E({ok:!0,item:s,message:`${n==="archive"?"Archived":"Restored"} ${s.id}`},r.json)});return}if(n==="untag"){if(ne(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");D(c,()=>{let a=C(c),d=a.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let s=a.items[d],l=s.tags?.length??0;s.tags=(s.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),s.updated_at=new Date().toISOString(),a.items[d]=s,K(c,a),E({ok:!0,item:s,removed:l-s.tags.length,message:`Removed tag from ${s.id}`},r.json)});return}if(n==="upsert"){let a=r.title??t[1],d=r.content??t[2];D(c,()=>{let s=C(c),l=r.id?s.items.findIndex((k)=>k.id===r.id||k.short_id===r.id):-1,f=new Date().toISOString();if(l===-1){if(!a||!d)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let k=r.id??Ie(),h={id:k,short_id:nt(k),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};s.items.push(h),K(c,s),E({ok:!0,created:!0,item:h,message:`Upserted ${h.id}`},r.json);return}let g=s.items[l];if(a!==void 0)g.title=a;if(d!==void 0)g.content=d;if(r.url!==void 0)g.url=r.url;if(r.tag!==void 0){if(g.tags=g.tags||[],!g.tags.map((k)=>k.toLowerCase()).includes(r.tag.toLowerCase()))g.tags.push(r.tag)}g.updated_at=f,s.items[l]=g,K(c,s),E({ok:!0,created:!1,item:g,message:`Upserted ${g.id}`},r.json)});return}if(n==="delete"){if(ne(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");D(c,()=>{let a=C(c),d=a.items.length;a.items=a.items.filter((l)=>l.id!==r.id&&l.short_id!==r.id);let s=d!==a.items.length;if(K(c,a),!s)throw Error(`Item not found: ${r.id}`);H("info","Item deleted",{id:r.id}),E({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let a=r.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");D(c,()=>{let d=C(c);if(a==="jsonl")for(let s of d.items)console.log(JSON.stringify(s));else E({ok:!0,items:d.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");D(c,()=>{let a=C(c),d=a.items.length;if(r.olderThan!==void 0){let l=new Date;l.setDate(l.getDate()-r.olderThan),a.items=a.items.filter((f)=>new Date(f.created_at)>=l)}if(r.empty)a.items=a.items.filter((l)=>l.content.trim().length>0);let s=d-a.items.length;K(c,a),H("info","Prune completed",{pruned:s,remaining:a.items.length}),E({ok:!0,pruned:s,remaining:a.items.length,message:`Pruned ${s} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");D(c,()=>{let a=C(c),d=new Set,s=a.items.length;a.items=a.items.filter((f)=>{let g=`${f.title}\x00${f.content}`;if(d.has(g))return!1;return d.add(g),!0});let l=s-a.items.length;K(c,a),H("info","Dedupe completed",{removed:l,remaining:a.items.length}),E({ok:!0,removed:l,remaining:a.items.length,message:`Dedupe removed ${l} duplicate(s)`},r.json)});return}if(n==="stats"){D(c,()=>{let a=C(c),d=a.items.filter((y)=>!y.archived),s=d.length,l=a.items.length-s,f=d.filter((y)=>y.url).length,g=d.filter((y)=>y.tags&&y.tags.length>0).length,k=s>0?d.map((y)=>y.created_at).sort()[0]:null,h=s>0?d.map((y)=>y.created_at).sort()[s-1]:null,m={};for(let y of d)for(let p of y.tags||[])m[p]=(m[p]||0)+1;let x=Object.entries(m).sort((y,p)=>p[1]-y[1]).slice(0,5).map(([y,p])=>({tag:y,count:p}));E({ok:!0,total:s,archived:l,with_url:f,with_tags:g,oldest:k,newest:h,top_tags:x,message:`${s} items | ${f} with URL | ${g} with tags`},r.json)});return}let u=Qi(t[0]),_=u?` Did you mean '${u}'?`:"";throw H("warn","Unknown command",{input:t[0],suggestion:u}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)is(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);H("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{Qi as suggestCommand,ns as sortItems,is as run,Ji as parseArgs};
|
|
599
|
+
--empty Remove items with empty content`)}function gs(e){if(e==="add"){console.log("Usage: open-knowledge add <title> <content> [--url <url>] [-t <tag>] [--json]");return}if(e==="list"||e==="ls"){console.log("Usage: open-knowledge list|ls [--format table|json] [-p <page>] [-l <limit>] [-s <search>] [-t <tag>] [--sort created|title] [--desc] [--json]");return}if(e==="get"){console.log("Usage: open-knowledge get --id <id> [--json]");return}if(e==="update"||e==="edit"){console.log("Usage: open-knowledge update|edit --id <id> [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="archive"){console.log("Usage: open-knowledge archive --id <id> [--json]");return}if(e==="restore"||e==="unarchive"){console.log("Usage: open-knowledge restore|unarchive --id <id> [--json]");return}if(e==="upsert"){console.log("Usage: open-knowledge upsert [title] [content] [--id <id>] [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="untag"){console.log("Usage: open-knowledge untag --id <id> -t <tag> [--json]");return}if(e==="delete"||e==="rm"){console.log("Usage: open-knowledge delete|rm --id <id> -y [--json]");return}if(e==="export"){console.log("Usage: open-knowledge export [--format jsonl] [--json]");return}if(e==="prune"){console.log("Usage: open-knowledge prune --yes [--older-than <days>] [--empty] [--json]");return}if(e==="dedupe"){console.log("Usage: open-knowledge dedupe --yes [--json]");return}if(e==="stats"){console.log("Usage: open-knowledge stats [--json]");return}if(e==="paths"){console.log("Usage: open-knowledge paths [--scope local|global|project] [--json]");return}if(e==="storage"){console.log("Usage: open-knowledge storage status|validate [--scope local|global|project] [--json]");return}if(e==="db"){console.log("Usage: open-knowledge db init|stats [--scope local|global|project] [--json]");return}if(e==="wiki"){console.log("Usage: open-knowledge wiki init [--scope local|global|project] [--json]");return}if(e==="source"){console.log("Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]");return}if(e==="ingest"){console.log("Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]");return}if(e==="reindex"){console.log("Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]");return}if(e==="search"){console.log("Usage: open-knowledge search <query> [--context] [--semantic] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="web"){console.log("Usage: open-knowledge web search <query> [--provider openai|anthropic] [--model provider:model] [--domain <domain>] [--file-results] [--fake] [--scope local|global|project] [--json]");return}if(e==="ask"||e==="build"||e==="knowledge"){console.log("Usage: open-knowledge ask|build <prompt> [--generate] [--semantic] [--model default|provider:model] [--approve-write] [--scope local|global|project] [--json]");return}if(e==="embeddings"){console.log("Usage: open-knowledge embeddings status|index|search [query] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="providers"){console.log("Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]");return}if(e==="safety"){console.log("Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]");return}fs()}function ps(e){if(e.noColor||process.env.NO_COLOR)return!1;if(process.env.FORCE_COLOR)return!0;return process.stdout.isTTY===!0}function E(e,t,r){if(t){console.log(JSON.stringify(e,null,2));return}if(typeof e==="string"){console.log(e);return}console.log(e.message??JSON.stringify(e,null,2))}function ae(e){if(!e.id)throw Error("Missing required --id. Example: open-knowledge get --id <id>")}function ms(e,t){let r=t.sort??"created";if(r!=="created"&&r!=="title")throw Error("Invalid --sort value. Use 'created' or 'title'.");let n=[...e].sort((i,s)=>{if(r==="title")return i.title.localeCompare(s.title);return i.created_at.localeCompare(s.created_at)});if(t.desc)n.reverse();return{sorted:n,sort:r,direction:t.desc?"desc":"asc"}}async function hs(e){let{positional:t,flags:r}=cs(e);if(J("debug","CLI invoked",{command:t[0],flags:{json:r.json,store:r.store}}),r.version){console.log(r.json?JSON.stringify({name:oe.name,version:oe.version},null,2):`${oe.name} ${oe.version}`);return}if(r.completions){let a=r.completions;if(a==="bash")console.log('_open_knowledge() { local cur; cur="${COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --semantic --context --generate --approve-write --provider --domain --file-results --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge');else if(a==="zsh")console.log(`#compdef open-knowledge
|
|
600
|
+
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--file-results)--file-results" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--provider)--provider[provider]:" "(--domain)--domain[domain]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l provider; complete -c open-knowledge -l domain; complete -c open-knowledge -l file-results; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=ds(t[0]),i=1;if(_s()&&n&&!gr.includes(n))n="ask",i=0;if(!n||r.help||n==="help"){gs(t[1]);return}let s=_r({scope:r.scope}),c=r.store;if(!c)if(r.scope==="project"||r.scope==="local")c=s.jsonStorePath();else c=Pe();if(n==="paths"){E(s.paths(),r.json);return}if(n==="storage"){let a=t[1]??"status";if(a==="status"){let d=s.storageContract(),o=s.validateStorage();E({ok:o.ok,...d,validation:o,message:`${d.storage_type} artifact storage at ${d.artifact_store.uri_prefix}`},r.json);return}if(a==="validate"){let d=s.validateStorage();E({ok:d.ok,validation:d,message:d.ok?"Storage contract valid":`Storage contract invalid: ${d.errors.join("; ")}`},r.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(n==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let o=s.initDb();E({ok:!0,...o,message:`Initialized ${o.path}`},r.json);return}let d=s.dbStats();E({ok:!0,path:s.workspace.knowledgeDbPath,...d,message:`knowledge.db schema v${d.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let d=await s.initWiki();E({ok:!0,...d,message:`Initialized wiki layout in ${s.workspace.home}`},r.json);return}if(n==="safety"){let a=t[1]??"status",d=s.ensureWorkspace(),o=s.safetyPolicy();s.initDb();let l=w(d.knowledgeDbPath);try{if(a==="status"){E({ok:!0,mode:o.mode,workspace:d.home,allow_write_roots:o.allowWriteRoots,read_only_source_access:o.readOnlySourceAccess,network:o.network,redaction:o.redaction,approvals:o.approvals,message:`Safety policy: ${o.mode}`},r.json);return}if(a==="check"){let f=t[2]??"generated_write",g=t[3]??null,k;try{if(f==="web_search")Y(o),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!g)throw Error("safety check s3_read requires an s3:// target.");q(g,o),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"}}else k=Bt(l,o,f,g);A(l,{event_type:"safety_check",action:f,target_uri:g,decision:k.decision==="allow"?"allow":"requires_approval",metadata:k}),E({ok:!0,...k,message:`Safety check ${k.decision}`},r.json);return}catch(h){throw A(l,{event_type:"safety_check",action:f,target_uri:g,decision:"deny",metadata:{error:h instanceof Error?h.message:String(h)}}),h}}if(a==="approve"){let f=t[2]??"generated_write",g=t[3]??null,k=Ht(l,{action:f,target_uri:g,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});A(l,{event_type:"approval",action:f,target_uri:g,decision:"allow",metadata:{approval_id:k.id}}),E({ok:!0,...k,action:f,target_uri:g,message:`Approved ${f}`},r.json);return}if(a==="audit"){let f=l.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((g)=>({id:g.id,event_type:g.event_type,action:g.action,target_uri:g.target_uri,decision:g.decision,metadata:JSON.parse(g.metadata_json),created_at:g.created_at}));E({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(a==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let g=ve(f,o);if(g.findings.length>0)we(l,{source_uri:"safety://redact",findings:g.findings,metadata:{command:"safety redact"}});A(l,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:g.findings.length>0?"redacted":"allow",metadata:{findings:g.findings.length}}),E({ok:!0,text:g.text,findings:g.findings,message:`Redacted ${g.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{l.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge source resolve <source-ref>");let o=await s.resolveSource(d,{purpose:r.purpose,limit:r.limit});E({ok:!0,...o,message:o.resolved?`Resolved ${o.source_ref} (${o.content.chunks_returned}/${o.content.chunks_total} chunks)`:`Source not indexed: ${d}`},r.json);return}if(n==="ingest"){let a=t[1]??"";if(a==="manifest"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let o=await s.ingestManifest(d);E({ok:!0,...o,message:`Ingested ${o.items_seen} manifest item(s)`},r.json);return}if(a==="source"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest source <source-ref>");let o=await s.ingestSource(d,r.purpose);E({ok:!0,...o,message:`Ingested source ${o.source_ref} (${o.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let o=await s.consumeOutbox(d);E({ok:!0,...o,message:`Consumed ${o.events_seen} outbox event(s)`},r.json);return}if(n==="embeddings"){let a=t[1]??"status";if(a==="status"){let d=s.embeddingStatus();E({ok:!0,...d,message:`${d.total_vector_entries} vector index entries`},r.json);return}if(a==="index"){let d=await s.indexEmbeddings({limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`Embedded ${d.chunks_embedded} chunk(s)`},r.json);return}if(a==="search"){let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge embeddings search <query>");let o=await s.semanticSearch({query:d,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...o,message:`${o.results.length} semantic result(s)`},r.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(n==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");if(r.context){let o=await s.retrieveContext({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...o,message:`${o.excerpts.length} context excerpt(s)`},r.json);return}let d=await s.search({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`${d.results.length} search result(s)`},r.json);return}if(n==="web"){if((t[1]??"search")!=="search")throw Error("Invalid web action. Use 'search'.");let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge web search <query>");let o=await s.webSearch({query:d,limit:r.limit,modelRef:r.model,provider:r.provider,domains:r.domain,fake:r.fake,fileResults:r.fileResults});E({ok:!0,...o,message:`${o.sources.length} web source(s)`},r.json);return}if(n==="ask"||n==="build"){let a=t.slice(i).join(" ");if(!a)throw Error("Usage: open-knowledge ask <prompt>");let d=await s.runPrompt({prompt:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake,generate:r.generate,approveWrite:r.approveWrite});E({ok:!0,...d,message:d.generated?"Generated answer with citations":"Prepared citation context draft"},r.json);return}if(n==="providers"){let a=t[1]??"status";if(a==="status"){let d=s.providerStatus(),o=d.providers.filter((l)=>l.configured).length;E({ok:!0,...d,message:`${o}/${d.providers.length} provider credential(s) configured`},r.json);return}if(a==="models"){let d=s.modelRegistry();E({ok:!0,models:d,message:`${d.length} model alias(es)`},r.json);return}if(a==="check"){let d=t[2]??"default",o=$(d,s.config()),l=U(o),f=B(l.provider,s.config());E({ok:!0,target:d,model_ref:o,provider:l.provider,model:l.model,credential:f,message:`${l.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(De(c),n==="add"){let a=t[1],d=t[2];if(!a||!d)throw Error("Usage: open-knowledge add <title> <content>");P(c,()=>{let o=C(c),l={id:Ue(),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};o.items.push(l),M(c,o),J("info","Item added",{id:l.id,title:l.title}),E({ok:!0,item:l,message:`Added ${l.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");P(c,()=>{let a=C(c),d=Number.isFinite(r.page)&&r.page>0?r.page:1,o=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,l=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",g=r.format==="table"||!r.json&&!r.format&&ps(r),k=r.json||r.format==="json",h=a.items;if(r.archived)h=h.filter((b)=>b.archived===!0);else if(!r.includeArchived)h=h.filter((b)=>!b.archived);if(l)h=h.filter((b)=>b.title.toLowerCase().includes(l)||b.content.toLowerCase().includes(l));if(f)h=h.filter((b)=>b.tags&&b.tags.map((Ae)=>Ae.toLowerCase()).includes(f));let{sorted:m,sort:S,direction:y}=ms(h,r),p=(d-1)*o,x=m.slice(p,p+o),N=Math.max(1,Math.ceil(m.length/o));if(k){E({ok:!0,page:d,limit:o,total:m.length,total_pages:N,sort:S,direction:y,items:x},!0);return}if(x.length===0){E(`No items found (search=${l||"none"}, tag=${f||"none"})`,!1);return}if(g){let b=(F)=>F,Ae=`${b("ID")} ${b("TITLE")} ${b("CREATED")} ${b("URL")} ${b("TAGS")}`;console.log(Ae);for(let F of x)console.log(`${F.id} ${b(F.title)} ${F.created_at} ${F.url?b(F.url):""} ${F.tags?.length?b(`[${F.tags.join(", ")}]`):""}`);console.log(`Page ${d}/${N} | showing ${x.length} of ${m.length} | sort=${S} ${y} | search=${l||"none"} | tag=${f||"none"}`)}else{for(let b of x)console.log(`${b.id} ${b.title} ${b.created_at}${b.url?` ${b.url}`:""}${b.tags?.length?` [${b.tags.join(", ")}]`:""}`);console.log(`Page ${d}/${N} | showing ${x.length} of ${m.length} | sort=${S} ${y} | search=${l||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){ae(r),P(c,()=>{let d=C(c).items.find((o)=>o.id===r.id||o.short_id===r.id);if(!d)throw Error(`Item not found: ${r.id}`);E({ok:!0,item:d,message:`${d.id}: ${d.title}`},r.json)});return}if(n==="update"){ae(r),P(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];if(r.title!==void 0)o.title=r.title;if(r.content!==void 0)o.content=r.content;if(r.url!==void 0)o.url=r.url;if(r.tag!==void 0){if(o.tags=o.tags||[],!o.tags.map((l)=>l.toLowerCase()).includes(r.tag.toLowerCase()))o.tags.push(r.tag)}o.updated_at=new Date().toISOString(),a.items[d]=o,M(c,a),E({ok:!0,item:o,message:`Updated ${o.id}`},r.json)});return}if(n==="archive"||n==="restore"){ae(r),P(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];o.archived=n==="archive",o.updated_at=new Date().toISOString(),a.items[d]=o,M(c,a),E({ok:!0,item:o,message:`${n==="archive"?"Archived":"Restored"} ${o.id}`},r.json)});return}if(n==="untag"){if(ae(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");P(c,()=>{let a=C(c),d=a.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d],l=o.tags?.length??0;o.tags=(o.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),o.updated_at=new Date().toISOString(),a.items[d]=o,M(c,a),E({ok:!0,item:o,removed:l-o.tags.length,message:`Removed tag from ${o.id}`},r.json)});return}if(n==="upsert"){let a=r.title??t[1],d=r.content??t[2];P(c,()=>{let o=C(c),l=r.id?o.items.findIndex((k)=>k.id===r.id||k.short_id===r.id):-1,f=new Date().toISOString();if(l===-1){if(!a||!d)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let k=r.id??Ue(),h={id:k,short_id:at(k),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};o.items.push(h),M(c,o),E({ok:!0,created:!0,item:h,message:`Upserted ${h.id}`},r.json);return}let g=o.items[l];if(a!==void 0)g.title=a;if(d!==void 0)g.content=d;if(r.url!==void 0)g.url=r.url;if(r.tag!==void 0){if(g.tags=g.tags||[],!g.tags.map((k)=>k.toLowerCase()).includes(r.tag.toLowerCase()))g.tags.push(r.tag)}g.updated_at=f,o.items[l]=g,M(c,o),E({ok:!0,created:!1,item:g,message:`Upserted ${g.id}`},r.json)});return}if(n==="delete"){if(ae(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");P(c,()=>{let a=C(c),d=a.items.length;a.items=a.items.filter((l)=>l.id!==r.id&&l.short_id!==r.id);let o=d!==a.items.length;if(M(c,a),!o)throw Error(`Item not found: ${r.id}`);J("info","Item deleted",{id:r.id}),E({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let a=r.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");P(c,()=>{let d=C(c);if(a==="jsonl")for(let o of d.items)console.log(JSON.stringify(o));else E({ok:!0,items:d.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");P(c,()=>{let a=C(c),d=a.items.length;if(r.olderThan!==void 0){let l=new Date;l.setDate(l.getDate()-r.olderThan),a.items=a.items.filter((f)=>new Date(f.created_at)>=l)}if(r.empty)a.items=a.items.filter((l)=>l.content.trim().length>0);let o=d-a.items.length;M(c,a),J("info","Prune completed",{pruned:o,remaining:a.items.length}),E({ok:!0,pruned:o,remaining:a.items.length,message:`Pruned ${o} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");P(c,()=>{let a=C(c),d=new Set,o=a.items.length;a.items=a.items.filter((f)=>{let g=`${f.title}\x00${f.content}`;if(d.has(g))return!1;return d.add(g),!0});let l=o-a.items.length;M(c,a),J("info","Dedupe completed",{removed:l,remaining:a.items.length}),E({ok:!0,removed:l,remaining:a.items.length,message:`Dedupe removed ${l} duplicate(s)`},r.json)});return}if(n==="stats"){P(c,()=>{let a=C(c),d=a.items.filter((y)=>!y.archived),o=d.length,l=a.items.length-o,f=d.filter((y)=>y.url).length,g=d.filter((y)=>y.tags&&y.tags.length>0).length,k=o>0?d.map((y)=>y.created_at).sort()[0]:null,h=o>0?d.map((y)=>y.created_at).sort()[o-1]:null,m={};for(let y of d)for(let p of y.tags||[])m[p]=(m[p]||0)+1;let S=Object.entries(m).sort((y,p)=>p[1]-y[1]).slice(0,5).map(([y,p])=>({tag:y,count:p}));E({ok:!0,total:o,archived:l,with_url:f,with_tags:g,oldest:k,newest:h,top_tags:S,message:`${o} items | ${f} with URL | ${g} with tags`},r.json)});return}let u=ls(t[0]),_=u?` Did you mean '${u}'?`:"";throw J("warn","Unknown command",{input:t[0],suggestion:u}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)hs(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);J("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{ls as suggestCommand,ms as sortItems,hs as run,cs as parseArgs};
|