@hasna/knowledge 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -1
- package/bin/open-knowledge-mcp.js +2275 -1643
- package/bin/open-knowledge.js +105 -79
- package/docs/architecture/ai-native-knowledge-base.md +11 -2
- package/docs/architecture/hybrid-semantic-search.md +12 -2
- package/package.json +2 -1
- package/src/agent.ts +367 -0
- package/src/cli.ts +58 -7
- package/src/mcp.js +36 -0
- package/src/providers.ts +1 -1
- package/src/retrieval.ts +326 -0
- package/src/service.ts +20 -0
package/bin/open-knowledge.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
|
-
var
|
|
4
|
-
`);return t}function
|
|
3
|
+
var N=import.meta.require;import{readFileSync as ae,writeFileSync as se,existsSync as oe,renameSync as mr,unlinkSync as tt}from"fs";import{randomUUID as rt}from"crypto";import{existsSync as ur,mkdirSync as Se,readFileSync as lr,writeFileSync as _r}from"fs";import{homedir as Ve}from"os";import{dirname as fr,join as I,resolve as gr}from"path";var V=I(".hasna","apps","knowledge");function Re(){return I(Ve(),".open-knowledge","db.json")}function Oe(){return I(Ve(),".hasna","apps","knowledge")}function pr(e=process.cwd()){return gr(e,V)}function Y(e){return{home:e,configPath:I(e,"config.json"),jsonStorePath:I(e,"db.json"),knowledgeDbPath:I(e,"knowledge.db"),artifactsDir:I(e,"artifacts"),cacheDir:I(e,"cache"),exportsDir:I(e,"exports"),indexesDir:I(e,"indexes"),logsDir:I(e,"logs"),runsDir:I(e,"runs"),schemasDir:I(e,"schemas"),wikiDir:I(e,"wiki")}}function hr(){return{version:1,mode:"local",storage:{type:"local",artifacts_root:"artifacts"},sources:{preferred_ref:"open-files",allowed_schemes:["open-files","s3","file","https","http"]},providers:{default_model:"openai:gpt-5.2",aliases:{fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"},openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},embeddings:{default_model:"openai:text-embedding-3-small",dimensions:1536,batch_size:64,max_parallel_calls:4},safety:{network:{web_search_enabled:!1,s3_reads_enabled:!1,allowed_s3_buckets:[]},redaction:{enabled:!0},approvals:{generated_writes_require_approval:!0}}}}function Qe(e){let t=Y(e);Se(t.home,{recursive:!0});for(let r of[t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir])Se(r,{recursive:!0});if(!ur(t.configPath))_r(t.configPath,`${JSON.stringify(hr(),null,2)}
|
|
4
|
+
`);return t}function Ze(e,t=process.cwd()){if(e==="project"||e==="local")return Y(pr(t));return Y(Oe())}function ie(e){Se(fr(e),{recursive:!0})}function et(e){let t=lr(e,"utf8");return JSON.parse(t)}function Ne(){return Y(Oe()).jsonStorePath}function Ae(e){if(!oe(e))if(ie(e),e===Ne()&&oe(Re()))se(e,ae(Re(),"utf8"));else se(e,JSON.stringify({items:[]},null,2))}function kr(e){return`${e}.lock`}function Er(e,t){let i=Date.now();while(Date.now()-i<5000){try{if(!oe(e)){se(e,JSON.stringify({owner:t,ts:Date.now()}));return}let c=JSON.parse(ae(e,"utf8"));if(Date.now()-c.ts>1e4)tt(e)}catch{}let o=Date.now();while(Date.now()-o<50);}throw Error(`Could not acquire lock on ${e} after 5000ms`)}function yr(e,t){try{if(oe(e)){if(JSON.parse(ae(e,"utf8")).owner===t)tt(e)}}catch{}}function C(e){Ae(e);let t=ae(e,"utf8"),r=JSON.parse(t);if(!r||!Array.isArray(r.items))return{items:[]};return r}function K(e,t){let r=`${e}.tmp.${rt()}`;se(r,JSON.stringify(t,null,2)),mr(r,e)}function D(e,t){let r=rt(),n=kr(e);Er(n,r);try{return t()}finally{yr(n,r)}}function Ie(){return`k_${Date.now().toString(36)}_${Math.random().toString(36).slice(2,8)}`}function nt(e){return e.replace(/^k_/,"").slice(0,12)}import{Database as br}from"bun:sqlite";var vr=`
|
|
5
5
|
PRAGMA journal_mode = WAL;
|
|
6
6
|
PRAGMA foreign_keys = ON;
|
|
7
7
|
|
|
@@ -168,7 +168,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
168
168
|
|
|
169
169
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
170
170
|
VALUES (1, datetime('now'));
|
|
171
|
-
`,
|
|
171
|
+
`,wr=`
|
|
172
172
|
DROP TABLE IF EXISTS chunks_fts;
|
|
173
173
|
|
|
174
174
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
@@ -181,7 +181,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
181
181
|
|
|
182
182
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
183
183
|
VALUES (2, datetime('now'));
|
|
184
|
-
`,
|
|
184
|
+
`,Tr=`
|
|
185
185
|
CREATE TABLE IF NOT EXISTS audit_events (
|
|
186
186
|
id TEXT PRIMARY KEY,
|
|
187
187
|
event_type TEXT NOT NULL,
|
|
@@ -212,7 +212,7 @@ CREATE INDEX IF NOT EXISTS idx_approval_gates_status ON approval_gates(status);
|
|
|
212
212
|
|
|
213
213
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
214
214
|
VALUES (3, datetime('now'));
|
|
215
|
-
`,
|
|
215
|
+
`,xr=`
|
|
216
216
|
CREATE TABLE IF NOT EXISTS vector_index_entries (
|
|
217
217
|
id TEXT PRIMARY KEY,
|
|
218
218
|
chunk_id TEXT NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
|
|
@@ -243,7 +243,8 @@ CREATE INDEX IF NOT EXISTS idx_vector_index_status ON vector_index_entries(statu
|
|
|
243
243
|
|
|
244
244
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
245
245
|
VALUES (4, datetime('now'));
|
|
246
|
-
`;function
|
|
246
|
+
`;function w(e){ie(e);let t=new br(e);return t.exec("PRAGMA foreign_keys = ON;"),t.exec("PRAGMA busy_timeout = 5000;"),t}function O(e){let t=w(e);try{if(t.exec(vr),Q(t)<2)t.exec(wr);if(Q(t)<3)t.exec(Tr);if(Q(t)<4)t.exec(xr);return{path:e,schema_version:Q(t)}}finally{t.close()}}function Q(e){return e.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0}function L(e,t){return e.query(`SELECT COUNT(*) AS n FROM ${t}`).get()?.n??0}function it(e){let t=w(e);try{return{schema_version:Q(t),sources:L(t,"sources"),source_revisions:L(t,"source_revisions"),chunks:L(t,"chunks"),wiki_pages:L(t,"wiki_pages"),citations:L(t,"citations"),indexes:L(t,"knowledge_indexes"),runs:L(t,"runs"),run_events:L(t,"run_events"),redaction_findings:L(t,"redaction_findings"),audit_events:L(t,"audit_events"),approval_gates:L(t,"approval_gates"),storage_objects:L(t,"storage_objects"),embeddings:L(t,"chunk_embeddings"),vector_entries:L(t,"vector_index_entries")}}finally{t.close()}}import{existsSync as Sr,mkdirSync as st,readFileSync as Rr,writeFileSync as Or}from"fs";import{dirname as Nr,join as Le,relative as Ar,sep as Ir}from"path";function Z(e){let t=e.replace(/\\/g,"/").trim();if(!t||t.startsWith("/"))throw Error(`Invalid artifact key: ${e}`);let r=t.split("/").filter(Boolean);if(r.length===0||r.some((n)=>n==="."||n===".."))throw Error(`Invalid artifact key: ${e}`);return r.join("/")}function Ce(e,t){let r=Ar(e,t);if(r.startsWith("..")||r===".."||r.startsWith(`..${Ir}`))throw Error(`Artifact path escapes root: ${t}`)}class ot{root;type="local";canRead=!0;canWrite=!0;constructor(e){this.root=e;st(e,{recursive:!0})}async put(e){let t=Z(e.key),r=Le(this.root,t);return Ce(this.root,r),st(Nr(r),{recursive:!0}),Or(r,e.body),{key:t,uri:`file://${r}`}}async getText(e){let t=Z(e),r=Le(this.root,t);return Ce(this.root,r),Rr(r,"utf8")}async exists(e){let t=Z(e),r=Le(this.root,t);return Ce(this.root,r),Sr(r)}}class at{options;type="s3";canRead=!0;canWrite=!0;client;constructor(e){this.options=e;this.client=e.client}async getClient(){if(this.client)return this.client;let[{S3Client:e},{fromIni:t}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]);return this.client=new e({region:this.options.region,credentials:this.options.profile?t({profile:this.options.profile}):void 0,maxAttempts:this.options.max_attempts}),this.client}objectKey(e){let t=Z(e),r=this.options.prefix?Z(this.options.prefix):"";return r?`${r}/${t}`:t}async put(e){let[{PutObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e.key);return await r.send(new t({Bucket:this.options.bucket,Key:n,Body:e.body,ContentType:e.content_type,Metadata:e.metadata,ServerSideEncryption:this.options.server_side_encryption,SSEKMSKeyId:this.options.kms_key_id})),{key:n,uri:`s3://${this.options.bucket}/${n}`}}async getText(e){let[{GetObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e),i=await r.send(new t({Bucket:this.options.bucket,Key:n}));if(!i.Body)return"";return await i.Body.transformToString()}async exists(e){let[{HeadObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e);try{return await r.send(new t({Bucket:this.options.bucket,Key:n})),!0}catch(i){let o=i instanceof Error?i.name:"";if(o==="NotFound"||o==="NoSuchKey"||o==="NotFoundError")return!1;throw i}}}function ct(e,t){if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)throw Error("S3 artifact storage requires storage.s3.bucket");return new at({bucket:e.storage.s3.bucket,prefix:e.storage.s3.prefix,region:e.storage.s3.region,profile:e.storage.s3.profile,max_attempts:e.storage.s3.max_attempts,server_side_encryption:e.storage.s3.server_side_encryption,kms_key_id:e.storage.s3.kms_key_id})}return new ot(t.artifactsDir)}import{randomUUID as Pt}from"crypto";import{randomUUID as Lr}from"crypto";var De={openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},Cr={openai:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!0,reasoning:!0,embeddings:!0},anthropic:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!1,reasoning:!0,embeddings:!1},deepseek:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!1,native_web_search:!1,reasoning:!0,embeddings:!1}},Dr={default:"openai:gpt-5.2",fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"};function ut(e){return e?.providers??{}}function ce(e,t){let r=ut(e)[t]??{};return{...De[t],...r}}function lt(e){let t=ut(e);return{...Dr,...t.default_model?{default:t.default_model}:{},...t.aliases??{}}}function M(e){let[t,...r]=e.split(":"),n=r.join(":");if(t!=="openai"&&t!=="anthropic"&&t!=="deepseek")throw Error(`Unsupported AI provider: ${t}`);if(!n)throw Error(`Invalid model ref: ${e}. Expected provider:model.`);return{provider:t,model:n}}function B(e,t){return lt(t)[e]??e}function Pe(e){let t=lt(e);return Object.entries(t).map(([r,n])=>{let i=M(n);return{alias:r,model_ref:n,provider:i.provider,model:i.model,default:r==="default",capabilities:Cr[i.provider]}})}function _t(e,t=process.env){return Object.keys(De).map((r)=>{let n=ce(e,r),i=Boolean(t[n.api_key_env]);return{provider:r,api_key_env:n.api_key_env,configured:i,source:i?"env":"missing",base_url:n.base_url??null,default_model:n.default_model}})}function ft(e,t=process.env){return{default_model:B("default",e),providers:_t(e,t),models:Pe(e)}}function ee(e,t,r=process.env){let n=_t(t,r).find((i)=>i.provider===e);if(!n)throw Error(`Unsupported AI provider: ${e}`);if(!n.configured)throw Error(`Missing ${n.api_key_env} for ${e}. Set the env var to use this provider.`);return n}async function Pr(e){if(e==="openai"){let{createOpenAI:r}=await import("@ai-sdk/openai");return r}if(e==="anthropic"){let{createAnthropic:r}=await import("@ai-sdk/anthropic");return r}let{createDeepSeek:t}=await import("@ai-sdk/deepseek");return t}async function Ur(e={}){let{createProviderRegistry:t}=await import("ai"),r=e.env??process.env,n={};for(let i of Object.keys(De)){let o=ce(e.config,i),c=r[o.api_key_env];if(!c)continue;let u=e.factories?.[i]??await Pr(i);n[i]=u({apiKey:c,baseURL:o.base_url})}return t(n)}async function gt(e,t={}){let r=B(e,t.config),n=M(r);return ee(n.provider,t.config,t.env),(await Ur(t)).languageModel(r)}function dt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return 0}function pt(e){let t=e.usage??{};return{provider:e.provider,model:e.model,input_tokens:dt(t,["inputTokens","promptTokens","input_tokens","prompt_tokens"]),output_tokens:dt(t,["outputTokens","completionTokens","output_tokens","completion_tokens"]),cost_usd:e.costUsd??0,metadata:{usage:t,provider_metadata:e.providerMetadata??{}}}}function ht(e,t){let r=`usage_${Lr()}`;return e.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
247
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[r,t.run_id??null,t.provider,t.model,t.input_tokens,t.output_tokens,t.cost_usd,JSON.stringify(t.metadata),t.created_at??new Date().toISOString()]),r}import{createHash as nn}from"crypto";function Ue(e){return["deleted","stale","invalidated","reindex_required"].includes((e??"").toLowerCase())}function W(e){let t=e.status??null;return{source_owner:"open-files",source_ref:e.source_ref??null,source_uri:e.source_uri??null,source_kind:e.source_kind??null,source_revision_id:e.source_revision_id??null,revision:e.revision??null,hash:e.hash??null,chunk_id:e.chunk_id??null,start_offset:e.start_offset??null,end_offset:e.end_offset??null,status:t,read_only:!0,citation_required:!0,resolver:e.resolver??null,stale:Ue(t)}}function je(e){return{source_owner:"open-files",generated_from:e.generated_from,artifact_key:e.artifact_key,source_refs:e.source_refs??[],read_only_sources:!0,citation_required:e.citation_required??!0,raw_source_bytes_stored_in_open_knowledge:!1}}function mt(e,t){return{...e,provenance:t}}import{createHash as yt}from"crypto";var jr="openai:text-embedding-3-small",bt=1536;function de(e){return e?.embeddings??{}}function kt(e,t){return`${e}_${yt("sha256").update(t).digest("hex").slice(0,20)}`}function Me(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function j(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function Et(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Ke(e){return Math.sqrt(e.reduce((t,r)=>t+r*r,0))}function Kr(e,t,r=Ke(t)){let n=Ke(e);if(n===0||r===0)return 0;let i=Math.min(e.length,t.length),o=0;for(let c=0;c<i;c+=1)o+=e[c]*t[c];return o/(n*r)}function Mr(e,t){let r=yt("sha256").update(e).digest();return Array.from({length:t},(n,i)=>{let o=r[i%r.length]/255;return Number((o*2-1).toFixed(6))})}async function Fr(e,t,r=process.env){ee("openai",t,r);let n=ce(t,"openai"),{createOpenAI:i}=await import("@ai-sdk/openai"),o=i({apiKey:r[n.api_key_env],baseURL:n.base_url});if(o.embeddingModel)return o.embeddingModel(e);if(o.textEmbedding)return o.textEmbedding(e);if(o.textEmbeddingModel)return o.textEmbeddingModel(e);throw Error("OpenAI provider does not expose an embedding model factory.")}function Fe(e,t){if(!e||e==="default"||e==="embedding")return de(t).default_model??jr;return e}async function vt(e,t={}){let r=Fe(t.modelRef,t.config),n=M(r);if(n.provider!=="openai")throw Error(`Embedding provider ${n.provider} is not supported yet. Use openai:text-embedding-3-small.`);let i=t.dimensions??de(t.config).dimensions??bt;if(t.fake)return{provider:n.provider,model:n.model,dimensions:i,vectors:e.map((a)=>Mr(a,i)),usage:{input_tokens:e.reduce((a,d)=>a+Math.max(1,Math.ceil(d.split(/\s+/).filter(Boolean).length*1.25)),0)}};let{embedMany:o}=await import("ai"),c=await Fr(n.model,t.config,t.env),u=await o({model:c,values:e,maxParallelCalls:t.maxParallelCalls??de(t.config).max_parallel_calls,providerOptions:{openai:{dimensions:i}}}),_=u.embeddings;return{provider:n.provider,model:n.model,dimensions:_[0]?.length??i,vectors:_,usage:{input_tokens:u.usage?.tokens??0}}}function Wr(e,t){if(t.sourceRevisionId)return e.query(`SELECT
|
|
247
248
|
c.id,
|
|
248
249
|
c.text,
|
|
249
250
|
c.token_count,
|
|
@@ -281,14 +282,14 @@ VALUES (4, datetime('now'));
|
|
|
281
282
|
ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
|
|
282
283
|
WHERE v.id IS NULL
|
|
283
284
|
ORDER BY c.created_at ASC, c.ordinal ASC
|
|
284
|
-
LIMIT ?`).all(t.provider,t.model,t.limit)}function
|
|
285
|
+
LIMIT ?`).all(t.provider,t.model,t.limit)}function Xr(e){let t=Me(e.metadata_json),r=t.provenance;if(r&&typeof r==="object"&&!Array.isArray(r))return r;return W({source_ref:j(t,["source_ref"]),source_uri:e.source_uri??j(t,["source_uri"]),source_kind:e.source_kind??j(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??j(t,["revision"]),hash:e.hash??j(t,["hash"]),chunk_id:e.id,start_offset:e.start_offset??Et(t,["start_offset"]),end_offset:e.end_offset??Et(t,["end_offset"]),status:j(t,["status"]),resolver:"open-files-read-only"})}function $r(e,t,r,n){let i=e.prepare(`
|
|
285
286
|
INSERT INTO chunk_embeddings (id, chunk_id, provider, model, dimensions, vector_json, created_at)
|
|
286
287
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
287
288
|
ON CONFLICT(chunk_id, provider, model) DO UPDATE SET
|
|
288
289
|
dimensions = excluded.dimensions,
|
|
289
290
|
vector_json = excluded.vector_json,
|
|
290
291
|
created_at = excluded.created_at
|
|
291
|
-
`),
|
|
292
|
+
`),o=e.prepare(`
|
|
292
293
|
INSERT INTO vector_index_entries (
|
|
293
294
|
id, chunk_id, source_revision_id, provider, model, dimensions, vector_json, vector_norm,
|
|
294
295
|
source_uri, source_ref, revision, hash, start_offset, end_offset, token_count, status,
|
|
@@ -310,10 +311,10 @@ VALUES (4, datetime('now'));
|
|
|
310
311
|
status = excluded.status,
|
|
311
312
|
metadata_json = excluded.metadata_json,
|
|
312
313
|
updated_at = excluded.updated_at
|
|
313
|
-
`);return e.transaction(()=>{for(let
|
|
314
|
+
`);return e.transaction(()=>{for(let u=0;u<t.length;u+=1){let _=t[u],a=r.vectors[u];if(!a)continue;let d=Me(_.metadata_json),s=Xr(_),l=s.source_ref??j(d,["source_ref"]),f=s.source_uri??_.source_uri??j(d,["source_uri"]),g=s.revision??_.revision??j(d,["revision"]),k=s.hash??_.hash??j(d,["hash"]),h=s.status??j(d,["status"])??"active",m=JSON.stringify(a);i.run(kt("emb",`${_.id}\x00${r.provider}\x00${r.model}`),_.id,r.provider,r.model,r.dimensions,m,n),o.run(kt("vec",`${_.id}\x00${r.provider}\x00${r.model}`),_.id,_.source_revision_id,r.provider,r.model,r.dimensions,m,Ke(a),f,l,g,k,s.start_offset,s.end_offset,_.token_count,h,JSON.stringify({...d,provenance:s,embedded_at:n}),n,n)}})(),t.length}async function wt(e){let t=Fe(e.modelRef,e.config),r=M(t);if(r.provider!=="openai")throw Error(`Embedding provider ${r.provider} is not supported yet.`);let n=(e.now??new Date).toISOString(),i=Math.max(1,Math.min(e.limit??100,1000));O(e.dbPath);let o=w(e.dbPath),c;try{c=Wr(o,{provider:r.provider,model:r.model,limit:i,sourceRevisionId:e.sourceRevisionId})}finally{o.close()}if(c.length===0)return{provider:r.provider,model:r.model,dimensions:e.dimensions??de(e.config).dimensions??bt,chunks_seen:0,chunks_embedded:0,embeddings_upserted:0,vector_entries_upserted:0,usage:{input_tokens:0}};let u=await vt(c.map((a)=>a.text),e),_=w(e.dbPath);try{let a=$r(_,c,u,n);return{provider:u.provider,model:u.model,dimensions:u.dimensions,chunks_seen:c.length,chunks_embedded:c.length,embeddings_upserted:a,vector_entries_upserted:a,usage:u.usage}}finally{_.close()}}function Tt(e){O(e);let t=w(e);try{let r=t.query("SELECT COUNT(*) AS n FROM chunk_embeddings").get()?.n??0,n=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0,i=t.query(`SELECT provider, model, dimensions, COUNT(*) AS entries, MAX(updated_at) AS updated_at
|
|
314
315
|
FROM vector_index_entries
|
|
315
316
|
GROUP BY provider, model, dimensions
|
|
316
|
-
ORDER BY provider, model`).all();return{total_embeddings:
|
|
317
|
+
ORDER BY provider, model`).all();return{total_embeddings:r,total_vector_entries:n,indexes:i}}finally{t.close()}}async function ue(e){let t=Fe(e.modelRef,e.config),r=M(t),n=Math.max(1,Math.min(e.limit??10,100)),i=await vt([e.query],e),o=i.vectors[0]??[];O(e.dbPath);let c=w(e.dbPath);try{let _=c.query(`SELECT
|
|
317
318
|
v.chunk_id,
|
|
318
319
|
c.text,
|
|
319
320
|
v.vector_json,
|
|
@@ -325,59 +326,7 @@ VALUES (4, datetime('now'));
|
|
|
325
326
|
v.metadata_json
|
|
326
327
|
FROM vector_index_entries v
|
|
327
328
|
JOIN chunks c ON c.id = v.chunk_id
|
|
328
|
-
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(
|
|
329
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[r,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),n]),r}function le(e,t){let n=t.created_at??new Date().toISOString();for(let r of t.findings)e.run(`INSERT INTO redaction_findings (id, source_uri, run_id, severity, finding_type, metadata_json, created_at)
|
|
330
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${Pe()}`,t.source_uri??null,t.run_id??null,r.severity,r.type,JSON.stringify({...t.metadata??{},start:r.start,end:r.end}),n]);return t.findings.length}function kt(e,t){let n=t.created_at??new Date().toISOString(),r=`approval_${Pe()}`;return e.run(`INSERT INTO approval_gates (id, action, target_uri, status, reason, approved_by, metadata_json, created_at, updated_at)
|
|
331
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[r,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),n,n]),{id:r,status:"approved"}}function Dn(e,t,n){let r=e.query(`SELECT id FROM approval_gates
|
|
332
|
-
WHERE action = ? AND status = 'approved' AND (target_uri IS NULL OR target_uri = ? OR ? IS NULL)
|
|
333
|
-
ORDER BY updated_at DESC LIMIT 1`).get(t,n??null,n??null);return Boolean(r)}function yt(e,t,n,r){let i=n==="generated_write"&&t.approvals.generatedWritesRequireApproval,s=!i||Dn(e,n,r);return{action:n,target_uri:r??null,approval_required:i,approved:s,decision:s?"allow":"requires_approval"}}function _e(e,t){return`${e}_${Pn("sha256").update(t).digest("hex").slice(0,20)}`}function H(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function v(e){return typeof e==="string"&&e.length>0?e:void 0}function Fn(e){let t=v(e.source_ref)??v(e.source_uri)??v(e.uri);if(t)return t;let n=v(e.file_id);if(n){let s=v(e.revision_id)??v(e.revision),d=`open-files://file/${encodeURIComponent(n)}`;return s?`${d}/revision/${encodeURIComponent(s)}`:d}let r=v(e.source_id),i=v(e.path);if(r&&i)return`open-files://source/${encodeURIComponent(r)}/path/${encodeURIComponent(i)}`;throw Error("Outbox event is missing source_ref, file_id, or source_id/path.")}function Wn(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Xn(e){return v(e.hash)??v(e.checksum)??v(e.sha256)??null}function $n(e,t,n){return v(e.revision_id)??v(e.revision)??v(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??n??null}function Bn(e){return(v(e.event)??v(e.type)??v(e.action)??v(e.change_type)??"changed").toLowerCase()}function Hn(e){let t=v(e.path);return v(e.title)??v(e.name)??(t?Kn(t):null)}function qn(e,t){let n=Fn(e),r=P(n),i=Xn(e);return{raw:e,eventType:Bn(e),sourceRef:n,sourceUri:Wn(n,r),kind:r.kind,title:Hn(e),revision:$n(e,r,i),hash:i,status:v(e.status)?.toLowerCase()??null,updatedAt:v(e.updated_at)??t,acl:e.permissions??e.acl??void 0}}function zn(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let n=JSON.parse(t);if(!Array.isArray(n))throw Error("Outbox array parse failed.");return n.map((r)=>{let i=H(r);if(!i)throw Error("Outbox array entries must be objects.");return i})}if(t.startsWith("{"))try{let n=JSON.parse(t),r=H(n);if(!r)throw Error("Outbox object parse failed.");if(Array.isArray(r.events))return r.events.map((i)=>{let s=H(i);if(!s)throw Error("Outbox events entries must be objects.");return s});if("source_ref"in r||"source_uri"in r||"file_id"in r)return[r]}catch(n){let r=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(r.length<=1)throw n;return r.map((i)=>{let s=H(JSON.parse(i));if(!s)throw Error("Outbox JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((n)=>n.trim().length>0).map((n)=>{let r=H(JSON.parse(n));if(!r)throw Error("Outbox JSONL entries must be objects.");return r})}async function Gn(e,t,n){let r=new URL(e),i=r.hostname,s=decodeURIComponent(r.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 outbox URI: ${e}`);if(n)F(e,n);let[{S3Client:d,GetObjectCommand:l},{fromIni:a}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),o=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new d({region:o?.region,credentials:o?.profile?a({profile:o.profile}):void 0,maxAttempts:o?.max_attempts}).send(new l({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function Jn(e,t,n){if(e.startsWith("s3://"))return Gn(e,t,n);if(!jn(e))throw Error(`Outbox not found: ${e}`);return Mn(e,"utf8")}function bt(e,t){let n={};if(e)try{n=H(JSON.parse(e))??{}}catch{n={}}return JSON.stringify({...n,...t})}function Yn(e,t,n){let r=_e("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
334
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
335
|
-
ON CONFLICT(uri) DO UPDATE SET
|
|
336
|
-
kind = excluded.kind,
|
|
337
|
-
title = COALESCE(excluded.title, sources.title),
|
|
338
|
-
updated_at = excluded.updated_at`,[r,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),n,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(v(t.raw.path))s.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[bt(i.metadata_json,s),t.acl===void 0?null:JSON.stringify(t.acl),t.acl===void 0?null:JSON.stringify(t.acl),t.updatedAt,i.id]),i.id}function Vn(e,t,n,r){if(!n.revision)return null;let i=_e("rev",`${t}\x00${n.revision}`),s={source_ref:n.sourceRef,source_uri:n.sourceUri,status:n.status,last_outbox_event:n.eventType,reindex_required:!0};return e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
339
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
340
|
-
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
341
|
-
hash = COALESCE(excluded.hash, source_revisions.hash),
|
|
342
|
-
metadata_json = excluded.metadata_json`,[i,t,n.revision,n.hash,v(n.raw.extracted_text_ref)??null,JSON.stringify(s),r]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,n.revision)?.id??null}function Qn(e,t,n){if(n.revision)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").all(t,n.revision).map((r)=>r.id);if(n.hash)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND hash = ?").all(t,n.hash).map((r)=>r.id);return e.query("SELECT id FROM source_revisions WHERE source_id = ?").all(t).map((r)=>r.id)}function Zn(e,t){let n=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t),r=0,i=0;for(let d of n){let l=e.query("SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?").get(d.id);r+=l?.n??0;let a=e.query("SELECT COUNT(*) AS n FROM vector_index_entries WHERE chunk_id = ?").get(d.id);i+=a?.n??0,e.run("DELETE FROM vector_index_entries WHERE chunk_id = ?",[d.id]),e.run("DELETE FROM chunk_embeddings WHERE chunk_id = ?",[d.id]),e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[d.id])}e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]);let s=e.query("SELECT metadata_json FROM source_revisions WHERE id = ?").get(t);return e.run("UPDATE source_revisions SET metadata_json = ? WHERE id = ?",[bt(s?.metadata_json,{reindex_required:!0,invalidated_at:new Date().toISOString()}),t]),{chunksDeleted:n.length,embeddingsDeleted:r,vectorEntriesDeleted:i}}function er(e,t){return t==="deleted"||["delete","deleted","remove","removed"].includes(e)}function tr(e){return["move","moved","rename","renamed","path_changed"].includes(e)}function nr(e){return["permission","permissions","permission_changed","acl_changed"].includes(e)}async function Tt(e){let t=(e.now??new Date).toISOString();if(e.safetyPolicy)X(e.dbPath,e.safetyPolicy);w(e.dbPath);let n=await Jn(e.input,e.config,e.safetyPolicy),r=zn(n),i=S(e.dbPath),s=`run_${Un()}`;try{return i.transaction(()=>{i.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
343
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:r.length}),t,t]);let d=new Set,l=new Set,a=0,o=0,c=0,u=0,_=0,f=0,m=0;return R(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:r.length,read_only:!0},created_at:t}),r.forEach((y,k)=>{let g=qn(y,t),b=Yn(i,g,t);d.add(b);let x=Vn(i,b,g,t);if(x)l.add(x);let p=Qn(i,b,g);for(let A of p){l.add(A);let h=Zn(i,A);a+=h.chunksDeleted,o+=h.embeddingsDeleted,c+=h.vectorEntriesDeleted,u+=1}if(er(g.eventType,g.status))_+=1;if(tr(g.eventType))f+=1;if(nr(g.eventType)||g.acl!==void 0)m+=1;i.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
344
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[_e("evt",`${s}\x00${k}\x00${g.sourceRef}\x00${g.eventType}`),s,"info",g.eventType,JSON.stringify({source_ref:g.sourceRef,source_uri:g.sourceUri,revision:g.revision,hash:g.hash,status:g.status,affected_revisions:p.length}),g.updatedAt])}),i.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
345
|
-
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[_e("usage",s),s,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),R(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:s,events:r.length,sources:d.size,revisions:l.size,chunks_deleted:a,embeddings_deleted:o,vector_entries_deleted:c},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:s,events_seen:r.length,sources_touched:d.size,revisions_touched:l.size,chunks_deleted:a,embeddings_deleted:o,vector_entries_deleted:c,stale_revisions:u,deleted_sources:_,moved_sources:f,permission_updates:m}})()}finally{i.close()}}import{createHash as rr}from"crypto";import{existsSync as ir,readFileSync as sr}from"fs";import{basename as or}from"path";function Ue(e,t){return`${e}_${rr("sha256").update(t).digest("hex").slice(0,20)}`}function q(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function T(e){return typeof e==="string"&&e.length>0?e:void 0}function ar(e){return typeof e==="number"&&Number.isFinite(e)?e:void 0}function cr(e){let t=T(e.source_ref)??T(e.source_uri)??T(e.uri);if(t)return t;let n=T(e.file_id);if(n){let s=T(e.revision_id)??T(e.revision),d=`open-files://file/${encodeURIComponent(n)}`;return s?`${d}/revision/${encodeURIComponent(s)}`:d}let r=T(e.source_id),i=T(e.path);if(r&&i)return`open-files://source/${encodeURIComponent(r)}/path/${encodeURIComponent(i)}`;throw Error("Manifest item is missing source_ref, file_id, or source_id/path.")}function ur(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function dr(e){let t=T(e.extracted_text)??T(e.text)??T(e.content_text)??T(e.markdown);if(t!==void 0)return t;let n=e.content;return typeof n==="string"?n:null}function lr(e){let t=T(e.extracted_text_ref)??T(e.extracted_text_uri)??T(e.text_ref);if(t)return t;let n=q(e.content);return T(n?.extracted_text_ref)??T(n?.extracted_text_uri)??null}function _r(e){let t=T(e.path);return T(e.title)??T(e.name)??(t?or(t):null)}function fr(e){return T(e.hash)??T(e.checksum)??T(e.sha256)??null}function gr(e,t,n){return T(e.revision_id)??T(e.revision)??T(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??n??T(e.updated_at)??"current"}function pr(e,t){let n={};for(let[r,i]of Object.entries(e)){if(["text","content","content_text","extracted_text","markdown"].includes(r))continue;n[r]=i}return n.source_ref=t.sourceRef,n.source_uri=t.sourceUri,n.status=t.status,n}function hr(e,t){let n=cr(e),r=P(n),i=ur(n,r),s=fr(e),d=T(e.status)??"active";return{raw:e,sourceRef:n,sourceUri:i,kind:r.kind,title:_r(e),revision:gr(e,r,s),hash:s,extractedTextUri:lr(e),text:dr(e),metadata:pr(e,{sourceRef:n,sourceUri:i,status:d}),acl:e.permissions??e.acl??{},status:d,updatedAt:T(e.updated_at)??t}}function mr(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let n=JSON.parse(t);if(!Array.isArray(n))throw Error("Manifest array parse failed.");return n.map((r)=>{let i=q(r);if(!i)throw Error("Manifest array entries must be objects.");return i})}if(t.startsWith("{"))try{let n=JSON.parse(t),r=q(n);if(!r)throw Error("Manifest object parse failed.");if(Array.isArray(r.items))return r.items.map((i)=>{let s=q(i);if(!s)throw Error("Manifest items entries must be objects.");return s});if("source_ref"in r||"source_uri"in r||"file_id"in r)return[r]}catch(n){let r=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(r.length<=1)throw n;return r.map((i)=>{let s=q(JSON.parse(i));if(!s)throw Error("Manifest JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((n)=>n.trim().length>0).map((n)=>{let r=q(JSON.parse(n));if(!r)throw Error("Manifest JSONL entries must be objects.");return r})}async function Er(e,t,n){let r=new URL(e),i=r.hostname,s=decodeURIComponent(r.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 manifest URI: ${e}`);if(n)F(e,n);let[{S3Client:d,GetObjectCommand:l},{fromIni:a}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),o=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new d({region:o?.region,credentials:o?.profile?a({profile:o.profile}):void 0,maxAttempts:o?.max_attempts}).send(new l({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function kr(e,t,n){if(e.startsWith("s3://"))return Er(e,t,n);if(!ir(e))throw Error(`Manifest not found: ${e}`);return sr(e,"utf8")}function yr(e,t,n){let r=e.replace(/\r\n/g,`
|
|
346
|
-
`);if(!r.trim())return[];let i=[],s=0;while(s<r.length){let d=Math.min(r.length,s+t),l=d;if(d<r.length){let o=r.lastIndexOf(`
|
|
347
|
-
|
|
348
|
-
`,d),c=r.lastIndexOf(". ",d),u=Math.max(o,c);if(u>s+Math.floor(t*0.5))l=u+(u===o?2:1)}let a=r.slice(s,l).trim();if(a)i.push({ordinal:i.length,text:a,startOffset:s,endOffset:l});if(l>=r.length)break;s=Math.max(0,l-n)}return i}function br(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Tr(e,t){let n=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t);for(let r of n)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[r.id]);return e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]),n.length}function vr(e,t,n){let r=Ue("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
349
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
350
|
-
ON CONFLICT(uri) DO UPDATE SET
|
|
351
|
-
kind = excluded.kind,
|
|
352
|
-
title = excluded.title,
|
|
353
|
-
metadata_json = excluded.metadata_json,
|
|
354
|
-
acl_json = excluded.acl_json,
|
|
355
|
-
updated_at = excluded.updated_at`,[r,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),n,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function xr(e,t,n,r){let i=Ue("rev",`${t}\x00${n.revision}`);e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
356
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
357
|
-
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
358
|
-
hash = excluded.hash,
|
|
359
|
-
extracted_text_uri = excluded.extracted_text_uri,
|
|
360
|
-
metadata_json = excluded.metadata_json`,[i,t,n.revision,n.hash,n.extractedTextUri,JSON.stringify(n.metadata),r]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,n.revision);if(!s)throw Error(`Failed to upsert source revision: ${n.sourceRef}`);return s.id}function Sr(e,t,n,r,i,s,d){if(!n.text||n.status.toLowerCase()==="deleted")return{chunksInserted:0,redactions:0};let l=de(n.text,d);if(l.findings.length>0)le(e,{source_uri:n.sourceUri,findings:l.findings,metadata:{source_ref:n.sourceRef,revision:n.revision},created_at:r}),R(e,{event_type:"redaction",action:"source_text_redact",target_uri:n.sourceUri,decision:"redacted",metadata:{findings:l.findings.length,source_ref:n.sourceRef,revision:n.revision},created_at:r});let a=yr(l.text,i,s);for(let o of a){let c=Ue("chk",`${t}\x00${o.ordinal}\x00${o.text}`),u=K({source_ref:n.sourceRef,source_uri:n.sourceUri,source_kind:n.kind,source_revision_id:t,revision:n.revision,hash:n.hash,chunk_id:c,start_offset:o.startOffset,end_offset:o.endOffset,status:n.status,resolver:"open-files-read-only"}),_=st({source_ref:n.sourceRef,source_uri:n.sourceUri,source_kind:n.kind,source_revision_id:t,revision:n.revision,hash:n.hash,status:n.status,path:T(n.raw.path)??null,mime:T(n.raw.mime)??T(n.raw.content_type)??null,size:ar(n.raw.size)??null},u);e.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
361
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[c,t,"source",o.ordinal,o.text,br(o.text),o.startOffset,o.endOffset,JSON.stringify(_),r]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[c,o.text,n.title??"",n.sourceUri])}return{chunksInserted:a.length,redactions:l.findings.length}}async function vt(e){let t=e.now??new Date;if(e.safetyPolicy)X(e.dbPath,e.safetyPolicy);w(e.dbPath);let n=await kr(e.input,e.config,e.safetyPolicy),r=mr(n);return je({dbPath:e.dbPath,items:r,sourceLabel:e.input,safetyPolicy:e.safetyPolicy,now:t,maxChunkChars:e.maxChunkChars,chunkOverlapChars:e.chunkOverlapChars})}async function je(e){let t=(e.now??new Date).toISOString(),n=e.maxChunkChars??4000,r=e.chunkOverlapChars??200;if(n<500)throw Error("maxChunkChars must be at least 500.");if(r<0||r>=n)throw Error("chunkOverlapChars must be less than maxChunkChars.");if(e.safetyPolicy)X(e.dbPath,e.safetyPolicy);w(e.dbPath);let i=S(e.dbPath);try{return i.transaction(()=>{let d=new Set,l=new Set,a=0,o=0,c=0,u=0;R(i,{event_type:"source_read",action:e.readAction??(e.sourceLabel.startsWith("s3://")?"s3_manifest_read":"local_manifest_read"),target_uri:e.sourceLabel,decision:"allow",metadata:{items:e.items.length,read_only:!0},created_at:t});for(let _ of e.items){let f=hr(_,t),m=vr(i,f,t),y=xr(i,m,f,t);if(d.add(m),l.add(y),f.text||f.status.toLowerCase()==="deleted")o+=Tr(i,y);let k=Sr(i,y,f,t,n,r,e.safetyPolicy);a+=k.chunksInserted,c+=k.redactions}return R(i,{event_type:"write",action:"knowledge_manifest_ingest",target_uri:e.dbPath,decision:"allow",metadata:{items:e.items.length,sources:d.size,revisions:l.size,chunks_inserted:a,redactions:c},created_at:t}),{path:e.sourceLabel,db_path:e.dbPath,items_seen:e.items.length,sources_upserted:d.size,revisions_upserted:l.size,chunks_inserted:a,chunks_deleted:o,redactions:c,skipped:u}})()}finally{i.close()}}import{createHash as Lr}from"crypto";import{existsSync as Cr,readFileSync as Dr}from"fs";import{basename as pe}from"path";function fe(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function $(e,t){for(let n of t){let r=e[n];if(typeof r==="string"&&r.length>0)return r}return null}function xt(e,t){for(let n of t){let r=e[n];if(typeof r==="number"&&Number.isFinite(r))return r}return null}function wr(e,t){let n=e.mode;if(typeof n==="string"&&n!=="read_only")throw Error(`Source resolver denied ${t}. Permission mode is ${n}, expected read_only.`);let r=e.denied_purposes;if(Array.isArray(r)&&r.includes(t))throw Error(`Source resolver denied ${t}. Purpose is explicitly denied.`);let i=e.allowed_purposes;if(Array.isArray(i)&&i.length>0&&!i.includes(t))throw Error(`Source resolver denied ${t}. Allowed purposes: ${i.join(", ")}`)}function Rr(e,t,n){if(!t)return n;try{let r=P(e);if(r.kind==="open-files"&&r.entity==="file")return`${e}/revision/${encodeURIComponent(t.revision)}`}catch{return n}return n}function Or(e,t,n){return e.query(`SELECT id, uri, kind, title, metadata_json, acl_json, updated_at
|
|
362
|
-
FROM sources
|
|
363
|
-
WHERE uri = ? OR uri = ?
|
|
364
|
-
ORDER BY CASE WHEN uri = ? THEN 0 ELSE 1 END
|
|
365
|
-
LIMIT 1`).get(t,n,t)??null}function Nr(e,t,n){if(n)return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
366
|
-
FROM source_revisions
|
|
367
|
-
WHERE source_id = ? AND revision = ?
|
|
368
|
-
LIMIT 1`).get(t,n)??null;return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
369
|
-
FROM source_revisions
|
|
370
|
-
WHERE source_id = ?
|
|
371
|
-
ORDER BY created_at DESC, revision DESC
|
|
372
|
-
LIMIT 1`).get(t)??null}function Ar(e,t){if(!t)return 0;return e.query("SELECT COUNT(*) AS n FROM chunks WHERE source_revision_id = ?").get(t)?.n??0}function Ir(e,t,n){if(!t||n<=0)return[];return e.query(`SELECT id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json
|
|
373
|
-
FROM chunks
|
|
374
|
-
WHERE source_revision_id = ?
|
|
375
|
-
ORDER BY ordinal ASC
|
|
376
|
-
LIMIT ?`).all(t,n)}async function ge(e){let t=e.purpose??"knowledge_answer",n=Math.max(0,Math.min(e.limit??10,100)),r=(e.now??new Date).toISOString(),i=P(e.sourceRef),s=gt(e.sourceRef,i),d=pt(e.sourceRef);if(e.safetyPolicy){if(!e.safetyPolicy.readOnlySourceAccess)throw Error("Safety policy denied source resolution.");X(e.dbPath,e.safetyPolicy)}w(e.dbPath);let l=S(e.dbPath);try{return l.transaction(()=>{let a=Or(l,s,e.sourceRef);if(!a)return R(l,{event_type:"source_read",action:"open_files_resolve_missing",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:s},created_at:r}),{source_ref:e.sourceRef,source_uri:s,purpose:t,read_only:!0,resolved:!1,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:null,revision:null,content:{mime:null,size:null,hash:null,text_available:!1,chunks_total:0,chunks_returned:0,char_count_returned:0,extracted_text_ref:null,bytes_available:!1,bytes_exposed:!1},chunks:[],citations:[]};let o=fe(a.metadata_json),c=fe(a.acl_json);try{wr(c,t)}catch(p){throw R(l,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"deny",metadata:{purpose:t,read_only:!0,source_uri:a.uri,error:p instanceof Error?p.message:String(p)},created_at:r}),p}let u=Nr(l,a.id,d),_=fe(u?.metadata_json),f=Ar(l,u?.id??null),m=Ir(l,u?.id??null,n),y=Rr(a.uri,u,e.sourceRef),k=m.map((p)=>{let A=fe(p.metadata_json),h={resolver:"open-files-read-only",mode:"local_catalog",purpose:t,read_only:!0,source_ref:$(A,["source_ref"])??y,source_uri:a.uri,source_revision_id:u?.id??null,revision:u?.revision??null,hash:u?.hash??$(A,["hash"]),chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,resolved_at:r},G=K({source_ref:h.source_ref,source_uri:h.source_uri,source_kind:a.kind,source_revision_id:h.source_revision_id,revision:h.revision,hash:h.hash,chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,status:$(A,["status"]),resolver:h.resolver});return{id:p.id,kind:p.kind,ordinal:p.ordinal,text:p.text,token_count:p.token_count,start_offset:p.start_offset,end_offset:p.end_offset,metadata:A,evidence:h,provenance:G}}),g=k.map((p)=>({source_ref:p.evidence.source_ref,source_uri:a.uri,chunk_id:p.id,quote:p.text.slice(0,500),start_offset:p.start_offset,end_offset:p.end_offset,evidence:p.evidence,provenance:p.provenance}));R(l,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:a.uri,revision:u?.revision??null,chunks_returned:k.length,chunks_total:f},created_at:r});let b=$(o,["mime","content_type"])??$(_,["mime","content_type"]),x=xt(o,["size","size_bytes"])??xt(_,["size","size_bytes"]);return{source_ref:y,source_uri:a.uri,purpose:t,read_only:!0,resolved:!0,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:{id:a.id,uri:a.uri,kind:a.kind,title:a.title,metadata:o,permissions:c,updated_at:a.updated_at},revision:u?{id:u.id,revision:u.revision,hash:u.hash,extracted_text_uri:u.extracted_text_uri,metadata:_,created_at:u.created_at,reindex_required:_.reindex_required===!0}:null,content:{mime:b,size:x,hash:u?.hash??$(o,["hash","checksum","sha256"]),text_available:f>0,chunks_total:f,chunks_returned:k.length,char_count_returned:k.reduce((p,A)=>p+A.text.length,0),extracted_text_ref:u?.extracted_text_uri??$(_,["extracted_text_ref","extracted_text_uri"]),bytes_available:!1,bytes_exposed:!1},chunks:k,citations:g}})()}finally{l.close()}}function z(e){return`sha256:${Lr("sha256").update(e).digest("hex")}`}function Pr(e){return e.replace(/<script[\s\S]*?<\/script>/gi," ").replace(/<style[\s\S]*?<\/style>/gi," ").replace(/<[^>]+>/g," ").replace(/ /g," ").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/\s+\n/g,`
|
|
377
|
-
`).replace(/\n\s+/g,`
|
|
378
|
-
`).replace(/[ \t]{2,}/g," ").trim()}async function Ur(e,t,n){let r=new URL(e),i=r.hostname,s=decodeURIComponent(r.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 source URI: ${e}`);if(n)F(e,n);let[{S3Client:d,GetObjectCommand:l},{fromIni:a}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),o=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new d({region:o?.region,credentials:o?.profile?a({profile:o.profile}):void 0,maxAttempts:o?.max_attempts}).send(new l({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function jr(e,t){if(t)ue(t);let n=await fetch(e,{headers:{accept:"text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5","user-agent":"@hasna/knowledge source-ingest"}});if(!n.ok)throw Error(`Web source read failed ${n.status}: ${e}`);let r=n.headers.get("content-type"),i=await n.text();return{text:r?.includes("html")?Pr(i):i,mime:r}}function he(e){if(e.kind==="file")return pe(e.path);if(e.kind==="s3")return pe(e.key);if(e.kind==="web")return pe(new URL(e.url).pathname)||e.url;return e.path?pe(e.path):e.id}async function St(e,t,n){if(e.kind==="file"){if(!Cr(e.path))throw Error(`Source file not found: ${e.path}`);let r=Dr(e.path,"utf8");return{text:r,contentSource:"file",title:he(e),mime:"text/plain",size:r.length,hash:z(r),revision:null,extractedTextRef:null,metadata:{path:e.path},permissions:{mode:"read_only"}}}if(e.kind==="s3"){let r=await Ur(e.uri,t,n);return{text:r,contentSource:"s3",title:he(e),mime:"text/plain",size:r.length,hash:z(r),revision:null,extractedTextRef:null,metadata:{bucket:e.bucket,key:e.key},permissions:{mode:"read_only"}}}if(e.kind==="web"){let r=await jr(e.url,n);return{text:r.text,contentSource:"web",title:he(e),mime:r.mime,size:r.text.length,hash:z(r.text),revision:null,extractedTextRef:null,metadata:{url:e.url},permissions:{mode:"read_only"}}}throw Error(`Direct source reading is not available for ${e.uri}`)}async function Mr(e,t,n){if(e.startsWith("open-files://"))throw Error("Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.");let r=P(e);return{text:(await St(r,t,n)).text,contentSource:"extracted_text_ref"}}async function Kr(e){let t=await ge({dbPath:e.dbPath,sourceRef:e.sourceRef,purpose:e.purpose??"knowledge_index",limit:100,safetyPolicy:e.safetyPolicy,now:e.now});if(!t.resolved)throw Error("Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.");if(t.revision?.extracted_text_uri&&!t.content.text_available){let r=await Mr(t.revision.extracted_text_uri,e.config,e.safetyPolicy);return{text:r.text,contentSource:r.contentSource,title:t.source?.title??null,mime:t.content.mime,size:r.text.length,hash:t.revision.hash??z(r.text),revision:t.revision.revision,extractedTextRef:t.revision.extracted_text_uri,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}if(t.chunks.length===0)throw Error("Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.");let n=t.chunks.map((r)=>r.text).join(`
|
|
379
|
-
|
|
380
|
-
`);return{text:n,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:n.length,hash:t.revision?.hash??z(n),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function Fr(e,t,n,r){let i=n.hash??z(n.text),s={...n.metadata,source_ref:e,content_source:n.contentSource,read_only:!0},d={source_ref:e,name:n.title??he(t),mime:n.mime??"text/plain",size:n.size??n.text.length,hash:i,revision:n.revision??i,status:"active",updated_at:new Date().toISOString(),permissions:{mode:"read_only",allowed_purposes:[r],...n.permissions},metadata:s,extracted_text_ref:n.extractedTextRef,extracted_text:n.text};if(t.kind==="open-files"){if(t.entity==="file")d.file_id=t.id;if(t.entity==="source")d.source_id=t.id,d.path=t.path}if(t.kind==="file")d.path=t.path;if(t.kind==="s3")d.path=t.key;if(t.kind==="web")d.url=t.url;return d}async function wt(e){let t=e.purpose??"knowledge_index",n=P(e.sourceRef),r=n.kind==="open-files"?await Kr(e):await St(n,e.config,e.safetyPolicy),i=Fr(e.sourceRef,n,r,t);return{...await je({dbPath:e.dbPath,items:[i],sourceLabel:e.sourceRef,readAction:"source_ref_ingest_read",safetyPolicy:e.safetyPolicy,now:e.now}),source_ref:e.sourceRef,content_source:r.contentSource,read_only:!0,hash:String(i.hash)}}function Ee(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function D(e,t){for(let n of t){let r=e[n];if(typeof r==="string"&&r.length>0)return r}return null}function Rt(e,t){for(let n of t){let r=e[n];if(typeof r==="number"&&Number.isFinite(r))return r}return null}function Ot(e){return Array.from(new Set(e))}function Wr(e){let t=e.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}_]+/gu)??[];return Ot(t.filter((n)=>n.length>0)).slice(0,16)}function Xr(e){if(e.length===0)return null;return e.map((t)=>`${t}*`).join(" OR ")}function $r(e){return e.replace(/[\\%_]/g,(t)=>`\\${t}`)}function Nt(e,t){return e.flatMap((n)=>Array.from({length:t},()=>`%${$r(n)}%`))}function Br(e,t){let n=Number.isFinite(e)?1/(1+Math.abs(e)):0,r=1/(1+t);return ke(Math.max(n,r))}function At(e,t){if(t.length===0)return 0;let n=t.filter((r)=>e.includes(r)).length;if(n===0)return 0;return ke(Math.min(0.85,0.35+n/t.length*0.5))}function Hr(e){return ke(Math.max(0,Math.min(1,(e+1)/2)))}function ke(e){return Number(e.toFixed(6))}function Z(e,t){let n=e.keyword??0,r=e.semantic??0,i=e.catalog??0,s=t?.chunk_id?0.05:0;return ke(Math.min(1,n*0.55+r*0.4+i*0.35+s))}function Me(e){let t=e.provenance;return t&&typeof t==="object"&&!Array.isArray(t)?t:null}function qr(e){let t=Ee(e.chunk_metadata_json),n=Me(t);if(n)return n;if(!e.source_revision_id&&!e.source_uri)return null;return K({source_ref:D(t,["source_ref"]),source_uri:e.source_uri??D(t,["source_uri"]),source_kind:e.source_kind??D(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??D(t,["revision"]),hash:e.hash??D(t,["hash"]),chunk_id:e.chunk_id,start_offset:e.start_offset??Rt(t,["start_offset"]),end_offset:e.end_offset??Rt(t,["end_offset"]),status:D(t,["status"]),resolver:"open-files-read-only"})}function zr(e,t,n){if(!t)return[];return e.query(`SELECT
|
|
329
|
+
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(r.provider,r.model).map((a)=>{let d=JSON.parse(a.vector_json),s=Me(a.metadata_json),l=s.provenance&&typeof s.provenance==="object"&&!Array.isArray(s.provenance)?s.provenance:null;return{chunk_id:a.chunk_id,score:Kr(o,d,a.vector_norm),text:a.text,source_uri:a.source_uri,source_ref:a.source_ref,revision:a.revision,hash:a.hash,provenance:l}}).sort((a,d)=>d.score-a.score).slice(0,n);return{provider:r.provider,model:r.model,dimensions:i.dimensions,query:e.query,results:_}}finally{c.close()}}function _e(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function P(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function xt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function St(e){return Array.from(new Set(e))}function qr(e){let t=e.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}_]+/gu)??[];return St(t.filter((r)=>r.length>0)).slice(0,16)}function Hr(e){if(e.length===0)return null;return e.map((t)=>`${t}*`).join(" OR ")}function Br(e){return e.replace(/[\\%_]/g,(t)=>`\\${t}`)}function Rt(e,t){return e.flatMap((r)=>Array.from({length:t},()=>`%${Br(r)}%`))}function zr(e,t){let r=Number.isFinite(e)?1/(1+Math.abs(e)):0,n=1/(1+t);return fe(Math.max(r,n))}function Ot(e,t){if(t.length===0)return 0;let r=t.filter((n)=>e.includes(n)).length;if(r===0)return 0;return fe(Math.min(0.85,0.35+r/t.length*0.5))}function Gr(e){return fe(Math.max(0,Math.min(1,(e+1)/2)))}function fe(e){return Number(e.toFixed(6))}function te(e,t){let r=e.keyword??0,n=e.semantic??0,i=e.catalog??0,o=t?.chunk_id?0.05:0;return fe(Math.min(1,r*0.55+n*0.4+i*0.35+o))}function We(e){let t=e.provenance;return t&&typeof t==="object"&&!Array.isArray(t)?t:null}function Jr(e){let t=_e(e.chunk_metadata_json),r=We(t);if(r)return r;if(!e.source_revision_id&&!e.source_uri)return null;return W({source_ref:P(t,["source_ref"]),source_uri:e.source_uri??P(t,["source_uri"]),source_kind:e.source_kind??P(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??P(t,["revision"]),hash:e.hash??P(t,["hash"]),chunk_id:e.chunk_id,start_offset:e.start_offset??xt(t,["start_offset"]),end_offset:e.end_offset??xt(t,["end_offset"]),status:P(t,["status"]),resolver:"open-files-read-only"})}function Yr(e,t,r){if(!t)return[];return e.query(`SELECT
|
|
381
330
|
chunks_fts.chunk_id,
|
|
382
331
|
c.kind AS chunk_kind,
|
|
383
332
|
c.wiki_page_id,
|
|
@@ -406,15 +355,88 @@ VALUES (4, datetime('now'));
|
|
|
406
355
|
LEFT JOIN wiki_pages wp ON wp.id = c.wiki_page_id
|
|
407
356
|
WHERE chunks_fts MATCH ?
|
|
408
357
|
ORDER BY rank ASC
|
|
409
|
-
LIMIT ?`).all(t,
|
|
358
|
+
LIMIT ?`).all(t,r)}function Nt(e,t){if(t.length===0)return"1 = 0";return t.map(()=>`(${e.map((n)=>`lower(COALESCE(${n}, '')) LIKE ? ESCAPE '\\'`).join(" OR ")})`).join(" OR ")}function Vr(e,t,r){let n=["path","title","artifact_uri","metadata_json"];return e.query(`SELECT id, path, title, artifact_uri, content_hash, status, metadata_json
|
|
410
359
|
FROM wiki_pages
|
|
411
|
-
WHERE status = 'active' AND (${
|
|
360
|
+
WHERE status = 'active' AND (${Nt(n,t)})
|
|
412
361
|
ORDER BY updated_at DESC
|
|
413
|
-
LIMIT ?`).all(...
|
|
362
|
+
LIMIT ?`).all(...Rt(t,n.length),r)}function Qr(e,t,r){let n=["kind","name","shard_key","artifact_uri","metadata_json"];return e.query(`SELECT id, kind, name, artifact_uri, shard_key, metadata_json
|
|
414
363
|
FROM knowledge_indexes
|
|
415
|
-
WHERE ${
|
|
364
|
+
WHERE ${Nt(n,t)}
|
|
416
365
|
ORDER BY updated_at DESC
|
|
417
|
-
LIMIT ?`).all(...
|
|
366
|
+
LIMIT ?`).all(...Rt(t,n.length),r)}function Zr(e,t){let r=_e(e.chunk_metadata_json),n=Jr(e),i=P(r,["source_ref"]),o=e.source_uri??P(r,["source_uri"]),c=Boolean(e.wiki_page_id),u={kind:c?"wiki_chunk":"source_chunk",id:e.chunk_id,title:c?e.wiki_title:e.source_title,text:e.text,score:0,scores:{keyword:t},source:o||i?{uri:o,ref:i,kind:e.source_kind??P(r,["source_kind"]),revision:e.revision??P(r,["revision"]),hash:e.hash??P(r,["hash"])}:null,citation:{chunk_id:e.chunk_id,start_offset:e.start_offset,end_offset:e.end_offset},artifact:c?{uri:e.wiki_artifact_uri,path:e.wiki_path,hash:e.wiki_content_hash,shard_key:e.wiki_path}:null,provenance:n,reasons:["keyword_match"]};return u.score=te(u.scores,u.citation),u}function en(e,t){let r=_e(e.metadata_json),n=Ot(`${e.path} ${e.title} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"wiki_page",id:e.id,title:e.title,text:null,score:0,scores:{catalog:n},source:null,citation:null,artifact:{uri:e.artifact_uri,path:e.path,hash:e.content_hash,shard_key:e.path},provenance:We(r),reasons:["wiki_catalog_match"]};return i.score=te(i.scores,i.citation),i}function tn(e,t){let r=_e(e.metadata_json),n=Ot(`${e.kind} ${e.name} ${e.shard_key??""} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"knowledge_index",id:e.id,title:e.name,text:null,score:0,scores:{catalog:n},source:null,citation:null,artifact:{uri:e.artifact_uri,path:P(r,["artifact_key"]),hash:P(r,["content_hash"]),shard_key:e.shard_key},provenance:We(r),reasons:["index_catalog_match"]};return i.score=te(i.scores,i.citation),i}function le(e,t){let r=`${t.kind}:${t.id}`,n=e.get(r);if(!n){e.set(r,t);return}n.scores={keyword:Math.max(n.scores.keyword??0,t.scores.keyword??0)||void 0,semantic:Math.max(n.scores.semantic??0,t.scores.semantic??0)||void 0,catalog:Math.max(n.scores.catalog??0,t.scores.catalog??0)||void 0},n.reasons=St([...n.reasons,...t.reasons]),n.text=n.text??t.text,n.title=n.title??t.title,n.source=n.source??t.source,n.citation=n.citation??t.citation,n.artifact=n.artifact??t.artifact,n.provenance=n.provenance??t.provenance,n.score=te(n.scores,n.citation)}function rn(e){let t={source_chunk:0,wiki_chunk:1,wiki_page:2,knowledge_index:3};return e.sort((r,n)=>{if(n.score!==r.score)return n.score-r.score;return t[r.kind]-t[n.kind]||r.id.localeCompare(n.id)})}async function ge(e){let t=e.query.trim();if(!t)throw Error("Search query is required.");let r=Math.max(1,Math.min(e.limit??10,100)),n=qr(t),i=Hr(n),o=e.semantic===!0||e.fake===!0||Boolean(e.modelRef),c=[],u=null,_=null,a=null,d=0,s=0,l=0,f=new Map;O(e.dbPath);let g=w(e.dbPath);try{let h=Yr(g,i,Math.max(r*3,20));d=h.length,h.forEach((y,p)=>le(f,Zr(y,zr(y.rank,p))));let m=Vr(g,n,Math.max(r,10)),x=Qr(g,n,Math.max(r,10));s=m.length+x.length,m.forEach((y)=>le(f,en(y,n))),x.forEach((y)=>le(f,tn(y,n)))}finally{g.close()}if(o)try{let h=await ue({dbPath:e.dbPath,query:t,limit:Math.max(r*3,20),config:e.config,env:e.env,modelRef:e.modelRef,dimensions:e.dimensions,fake:e.fake,batchSize:e.batchSize,maxParallelCalls:e.maxParallelCalls});u=h.provider,_=h.model,a=h.dimensions,l=h.results.length;for(let m of h.results){let x={kind:"source_chunk",id:m.chunk_id,title:null,text:m.text,score:0,scores:{semantic:Gr(m.score)},source:{uri:m.source_uri,ref:m.source_ref,kind:m.provenance?.source_kind??null,revision:m.revision,hash:m.hash},citation:{chunk_id:m.chunk_id,start_offset:m.provenance?.start_offset??null,end_offset:m.provenance?.end_offset??null},artifact:null,provenance:m.provenance,reasons:["semantic_match"]};x.score=te(x.scores,x.citation),le(f,x)}}catch(h){c.push(`semantic_search_failed: ${h instanceof Error?h.message:String(h)}`)}let k=rn(Array.from(f.values())).slice(0,r);return{query:t,limit:r,mode:{keyword:!0,catalog:!0,semantic:o},semantic_provider:u,semantic_model:_,semantic_dimensions:a,counts:{keyword_results:d,catalog_results:s,semantic_results:l,merged_results:k.length},warnings:c,results:k}}function At(e,t){return`${e}_${nn("sha256").update(t).digest("hex").slice(0,20)}`}function It(e){return e.normalize("NFKC").trim().replace(/\s+/g," ").toLowerCase()}function sn(e){return Array.from(new Set(It(e).match(/[\p{L}\p{N}_]+/gu)??[])).slice(0,16)}function on(e){return[e.title,e.text].filter(Boolean).join(" ").toLowerCase()}function an(e,t){if(t.length===0)return 0;let r=on(e),n=t.filter((i)=>r.includes(i)).length;return Number((n/t.length).toFixed(6))}function cn(e){if(!e)return!0;if("read_only"in e)return e.read_only===!0;if("read_only_sources"in e)return e.read_only_sources===!0;return!0}function Lt(e){if(!e)return!1;if("stale"in e&&e.stale)return!0;if("status"in e)return Ue(e.status);return!1}function dn(e){if(Lt(e.provenance))return 0;if(e.source?.hash||e.source?.revision)return 1;if(e.artifact?.hash)return 0.85;if(e.provenance&&"source_refs"in e.provenance&&e.provenance.source_refs.length>0)return 0.75;return 0.55}function un(e){if(e.citation?.chunk_id&&(e.source?.uri||e.artifact?.uri))return 1;if(e.provenance&&"citation_required"in e.provenance&&e.provenance.citation_required)return 0.75;if(e.artifact?.uri)return 0.65;return 0.35}function ln(e){if(e.kind==="wiki_chunk")return 0.85;if(e.kind==="source_chunk")return 0.8;if(e.kind==="wiki_page")return 0.65;return 0.55}function _n(e,t){let r={base_score:e.score,exact_score:an(e,t),citation_score:un(e),freshness_score:dn(e),authority_score:ln(e)},n=Math.min(1,r.base_score*0.65+r.exact_score*0.1+r.citation_score*0.1+r.freshness_score*0.1+r.authority_score*0.05),i=new Set(e.reasons);if(r.exact_score>0.5)i.add("exact_term");if(r.citation_score>=0.75)i.add("cited_source");if(r.freshness_score>=0.85)i.add("fresh_source");return{...e,score:Number(n.toFixed(6)),reasons:Array.from(i),rerank:{...r,final_score:Number(n.toFixed(6))}}}function Ct(e,t){let r=e.text??e.title;if(!r)return null;let n=r.replace(/\s+/g," ").trim();return n.length<=t?n:`${n.slice(0,Math.max(0,t-1)).trim()}...`}function fn(e){return{id:At("cite",`${e.kind}\x00${e.id}\x00${e.source?.uri??""}\x00${e.artifact?.uri??""}`),result_id:e.id,kind:e.kind,source_uri:e.source?.uri??null,source_ref:e.source?.ref??null,artifact_uri:e.artifact?.uri??null,artifact_path:e.artifact?.path??null,revision:e.source?.revision??null,hash:e.source?.hash??e.artifact?.hash??null,chunk_id:e.citation?.chunk_id??null,start_offset:e.citation?.start_offset??null,end_offset:e.citation?.end_offset??null,quote:Ct(e,500),provenance:e.provenance}}function gn(e,t,r){let n=Ct(e,r);if(!n)return null;return{id:At("excerpt",`${e.kind}\x00${e.id}`),result_id:e.id,citation_id:t.id,kind:e.kind,text:n,score:e.score}}function pe(e){return e.map(()=>"?").join(", ")}function pn(e,t){let r=t.map((u)=>u.citation?.chunk_id).filter((u)=>Boolean(u)),n=t.filter((u)=>u.kind==="wiki_page").map((u)=>u.id),i=[],o=[];if(r.length===0&&n.length===0)return{citations:i,backlinks:o};let c=w(e);try{if(r.length>0)i.push(...c.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
367
|
+
FROM citations
|
|
368
|
+
WHERE chunk_id IN (${pe(r)})
|
|
369
|
+
ORDER BY created_at DESC
|
|
370
|
+
LIMIT 50`).all(...r));if(n.length>0)i.push(...c.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
371
|
+
FROM citations
|
|
372
|
+
WHERE wiki_page_id IN (${pe(n)})
|
|
373
|
+
ORDER BY created_at DESC
|
|
374
|
+
LIMIT 50`).all(...n)),o.push(...c.query(`SELECT from_page_id, to_page_id, label
|
|
375
|
+
FROM wiki_backlinks
|
|
376
|
+
WHERE from_page_id IN (${pe(n)}) OR to_page_id IN (${pe(n)})
|
|
377
|
+
LIMIT 50`).all(...n,...n))}finally{c.close()}return{citations:i,backlinks:o}}async function he(e){let t=Math.max(200,Math.min(e.contextChars??1200,4000)),r=await ge(e),n=sn(r.query),i=[...r.warnings],o=new Set,c=new Set,_=r.results.filter((s)=>{if(!cn(s.provenance))return i.push(`permission_filtered: ${s.kind}:${s.id}`),o.add("Dropped a result because provenance was not read-only."),!1;if(Lt(s.provenance))return i.push(`stale_filtered: ${s.kind}:${s.id}`),c.add("Dropped a stale result whose source status requires reindexing."),!1;return!0}).map((s)=>_n(s,n)).sort((s,l)=>l.score-s.score||s.id.localeCompare(l.id)).slice(0,r.limit),a=_.map(fn),d=_.map((s,l)=>gn(s,a[l],t)).filter((s)=>Boolean(s));for(let s of _){if(s.provenance&&"read_only"in s.provenance&&s.provenance.read_only)o.add("All source-backed excerpts are read-only and citation-required.");if(s.rerank.freshness_score>=0.85)c.add("Fresh source revision/hash or artifact hash is present for top context.")}return{query:r.query,normalized_query:It(r.query),created_at:new Date().toISOString(),mode:r.mode,warnings:i,search_counts:r.counts,results:_,citations:a,excerpts:d,graph:pn(e.dbPath,_),notes:{permissions:Array.from(o),freshness:Array.from(c)}}}function Xe(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function qe(e){return`C${e+1}`}function hn(e,t){if(t.excerpts.length===0)return`No indexed knowledge matched the prompt: ${e}`;return[`Found ${t.excerpts.length} relevant knowledge excerpt(s) for: ${e}`,"",...t.excerpts.slice(0,5).map((n,i)=>{let o=t.citations.find((u)=>u.id===n.citation_id),c=o?.source_ref??o?.source_uri??o?.artifact_path??o?.artifact_uri??"unknown source";return`[${qe(i)}] ${n.text} (${c})`})].join(`
|
|
378
|
+
`)}function mn(e,t){let r=t.citations.map((i,o)=>({id:qe(o),source_ref:i.source_ref,source_uri:i.source_uri,artifact_path:i.artifact_path,revision:i.revision,hash:i.hash,quote:i.quote})),n=t.excerpts.map((i,o)=>({id:qe(o),kind:i.kind,text:i.text,score:i.score}));return[`Prompt: ${e}`,"","Use only the provided context. Cite claims with citation ids like [C1]. If context is insufficient, say what is missing.","",`Context excerpts:
|
|
379
|
+
${JSON.stringify(n,null,2)}`,"",`Citations:
|
|
380
|
+
${JSON.stringify(r,null,2)}`].join(`
|
|
381
|
+
`)}function kn(e,t){if(t.citations.length===0)return[];return[{kind:"answer_note",title:e.length>80?`${e.slice(0,77)}...`:e,citations:t.citations.map((r)=>r.id),requires_approval:!0}]}function En(e,t){let r=w(e);try{r.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
382
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[t.runId,"knowledge-prompt",t.prompt,t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.now])}finally{r.close()}}function $e(e,t){let r=w(e);try{r.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
383
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${Pt()}`,t.runId,t.level,t.event,JSON.stringify(t.metadata),t.now])}finally{r.close()}}function Dt(e,t){let r=w(e);try{r.run(`UPDATE runs
|
|
384
|
+
SET status = ?, provider = ?, model = ?, metadata_json = ?, updated_at = ?
|
|
385
|
+
WHERE id = ?`,[t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.runId])}finally{r.close()}}function yn(e,t,r,n,i,o,c={}){let u=w(e);try{ht(u,{run_id:t,provider:n,model:i,input_tokens:r.input_tokens,output_tokens:r.output_tokens,cost_usd:r.cost_usd,metadata:c,created_at:o})}finally{u.close()}}async function Ut(e){let t=e.prompt.trim();if(!t)throw Error("Knowledge prompt is required.");let r=(e.now??new Date).toISOString(),n=`run_${Pt()}`,i=B(e.modelRef??"default",e.config),o=M(i);O(e.dbPath),En(e.dbPath,{runId:n,prompt:t,status:e.generate?"running":"dry_run",provider:e.generate?o.provider:"local",model:e.generate?o.model:"context-draft",metadata:{semantic:e.semantic===!0||e.fake===!0||Boolean(e.modelRef),approve_write:e.approveWrite===!0,generated:e.generate===!0},now:r});let{prompt:c,generate:u,approveWrite:_,now:a,...d}=e,s=await he({...d,query:t});$e(e.dbPath,{runId:n,level:"info",event:"context_retrieved",metadata:{results:s.results.length,citations:s.citations.length,warnings:s.warnings},now:r});let l=hn(t,s),f=!1,g="local",k="context-draft",h={input_tokens:Xe(t)+s.excerpts.reduce((p,S)=>p+Xe(S.text),0),output_tokens:Xe(l),cost_usd:0},m=[...s.warnings];if(e.generate)try{if(e.fake)f=!0,g=o.provider,k=o.model,l=`Fake generated answer for: ${t}
|
|
386
|
+
|
|
387
|
+
${l}`;else{let{generateText:p}=await import("ai"),S=await gt(i,{config:e.config,env:e.env}),R=await p({model:S,system:"You answer company knowledge-base prompts using only provided context and citation ids.",prompt:mn(t,s)});f=!0,g=o.provider,k=o.model,l=R.text;let b=pt({provider:g,model:k,usage:R.usage,providerMetadata:R.providerMetadata});h={input_tokens:b.input_tokens,output_tokens:b.output_tokens,cost_usd:b.cost_usd}}}catch(p){throw $e(e.dbPath,{runId:n,level:"error",event:"answer_generation_failed",metadata:{message:p instanceof Error?p.message:String(p)},now:r}),Dt(e.dbPath,{runId:n,status:"failed",provider:o.provider,model:o.model,metadata:{generated:!1,error:p instanceof Error?p.message:String(p)},now:r}),p}let x=kn(t,s),y={approved:e.approveWrite===!0,durable_writes_performed:!1,reason:e.approveWrite?"Approval flag recorded; durable wiki writing is deferred to the wiki compile task.":"Dry-run mode: proposed wiki updates require approval before durable writes."};return $e(e.dbPath,{runId:n,level:"info",event:f?"answer_generated":"answer_drafted",metadata:{provider:g,model:k,proposed_updates:x.length,durable_writes_performed:!1},now:r}),yn(e.dbPath,n,h,g,k,r,{generated:f,citations:s.citations.length}),Dt(e.dbPath,{runId:n,status:f?"completed":"dry_run",provider:g,model:k,metadata:{generated:f,citations:s.citations.length,proposed_updates:x.length,approve_write:e.approveWrite===!0},now:r}),{run_id:n,prompt:t,generated:f,provider:g,model:k,answer:l,context:s,citations:s.citations,proposed_wiki_updates:x,write_policy:y,usage:h,warnings:m}}import{createHash as Ln,randomUUID as Cn}from"crypto";import{existsSync as Dn,readFileSync as Pn}from"fs";import{basename as Un}from"path";function jt(e,t){if(!e)throw Error(t);return e}function bn(e){let r=e.slice(13).split("/").filter(Boolean),n=r[0];if(n!=="file"&&n!=="source")throw Error("Invalid open-files ref. Expected open-files://file/<id>, open-files://file/<id>/revision/<revision_id>, or open-files://source/<id>/path/<path>.");let i=jt(r[1],"Invalid open-files ref. Missing id.");if(n==="file"){if(r.length===2)return{kind:"open-files",uri:e,entity:n,id:i};if(r[2]==="revision"&&r[3]&&r.length===4)return{kind:"open-files",uri:e,entity:n,id:i,revision_id:decodeURIComponent(r[3])};throw Error("Invalid open-files file ref. Expected open-files://file/<id>/revision/<revision_id>.")}let o=r.indexOf("path"),c=o>=0?decodeURIComponent(r.slice(o+1).join("/")):void 0;return{kind:"open-files",uri:e,entity:n,id:i,path:c}}function vn(e){let t=new URL(e),r=jt(t.hostname,"Invalid s3 ref. Missing bucket."),n=decodeURIComponent(t.pathname.replace(/^\/+/,""));if(!n)throw Error("Invalid s3 ref. Missing object key.");return{kind:"s3",uri:e,bucket:r,key:n}}function wn(e){let t=new URL(e);return{kind:"file",uri:e,path:decodeURIComponent(t.pathname)}}function Tn(e){let t=new URL(e);return{kind:"web",uri:e,url:t.toString()}}function U(e){if(e.startsWith("open-files://"))return bn(e);if(e.startsWith("s3://"))return vn(e);if(e.startsWith("file://"))return wn(e);if(e.startsWith("https://")||e.startsWith("http://"))return Tn(e);throw Error(`Unsupported source ref scheme: ${e}`)}function Kt(e,t=U(e)){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Mt(e){let t=U(e);return t.kind==="open-files"&&t.entity==="file"?t.revision_id??null:null}import{createHash as xn,randomUUID as He}from"crypto";import{relative as Sn,resolve as Wt,sep as Rn}from"path";function Ft(e){let t=process.env[e];return t==="1"||t==="true"||t==="yes"}function Xt(e,t){let r=e,n=new Set(r.safety?.network?.allowed_s3_buckets??[]);if(e.storage.type==="s3"&&e.storage.s3?.bucket)n.add(e.storage.s3.bucket);if(process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS)for(let i of process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.split(",").map((o)=>o.trim()).filter(Boolean))n.add(i);return{mode:e.mode,allowWriteRoots:[t.home,t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir].map((i)=>Wt(i)),readOnlySourceAccess:!0,network:{webSearchEnabled:r.safety?.network?.web_search_enabled??Ft("HASNA_KNOWLEDGE_WEB_SEARCH"),s3ReadsEnabled:r.safety?.network?.s3_reads_enabled??Ft("HASNA_KNOWLEDGE_ALLOW_S3_READS"),allowedS3Buckets:[...n].sort()},redaction:{enabled:r.safety?.redaction?.enabled??!0},approvals:{generatedWritesRequireApproval:r.safety?.approvals?.generated_writes_require_approval??!0}}}function On(e,t){let r=Sn(e,t);return r===""||!r.startsWith("..")&&r!==".."&&!r.startsWith(`..${Rn}`)}function $(e,t){let r=Wt(e);if(!t.allowWriteRoots.some((n)=>On(n,r)))throw Error(`Safety policy denied write outside .hasna/apps/knowledge: ${e}`)}function X(e,t){let n=new URL(e).hostname;if(!t.network.s3ReadsEnabled)throw Error("Safety policy denied S3 read. Set safety.network.s3_reads_enabled=true or HASNA_KNOWLEDGE_ALLOW_S3_READS=1.");if(!t.network.allowedS3Buckets.includes(n))throw Error(`Safety policy denied S3 bucket "${n}". Add it to safety.network.allowed_s3_buckets or HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.`)}function me(e){if(!e.network.webSearchEnabled)throw Error("Safety policy denied web search. Set safety.network.web_search_enabled=true or HASNA_KNOWLEDGE_WEB_SEARCH=1.")}var Nn=[{type:"private_key_block",severity:"high",regex:/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,replacement:"[REDACTED:private_key_block]"},{type:"secret_assignment",severity:"high",regex:/\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*['"]?[^'"\s]{8,}/gi,replacement:"[REDACTED:secret_assignment]"},{type:"openai_api_key",severity:"high",regex:/\bsk-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:openai_api_key]"},{type:"anthropic_api_key",severity:"high",regex:/\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:anthropic_api_key]"},{type:"aws_access_key_id",severity:"high",regex:/\bA(?:KIA|SIA)[A-Z0-9]{16}\b/g,replacement:"[REDACTED:aws_access_key_id]"}];function ke(e,t){if(t&&!t.redaction.enabled)return{text:e,findings:[]};let r=e,n=[];for(let i of Nn)r=r.replace(i.regex,(o,...c)=>{let u=typeof c.at(-2)==="number"?c.at(-2):r.indexOf(o);return n.push({type:i.type,severity:i.severity,start:Math.max(0,u),end:Math.max(0,u+o.length)}),i.replacement});return{text:r,findings:n}}function An(e){return`audit_${xn("sha256").update(`${e.event_type}\x00${e.action}\x00${e.target_uri??""}\x00${e.created_at??""}\x00${JSON.stringify(e.metadata??{})}\x00${He()}`).digest("hex").slice(0,24)}`}function A(e,t){let r=t.created_at??new Date().toISOString(),n=An({...t,created_at:r});return e.run(`INSERT INTO audit_events (id, event_type, action, target_uri, decision, metadata_json, created_at)
|
|
388
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function Ee(e,t){let r=t.created_at??new Date().toISOString();for(let n of t.findings)e.run(`INSERT INTO redaction_findings (id, source_uri, run_id, severity, finding_type, metadata_json, created_at)
|
|
389
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${He()}`,t.source_uri??null,t.run_id??null,n.severity,n.type,JSON.stringify({...t.metadata??{},start:n.start,end:n.end}),r]);return t.findings.length}function $t(e,t){let r=t.created_at??new Date().toISOString(),n=`approval_${He()}`;return e.run(`INSERT INTO approval_gates (id, action, target_uri, status, reason, approved_by, metadata_json, created_at, updated_at)
|
|
390
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function In(e,t,r){let n=e.query(`SELECT id FROM approval_gates
|
|
391
|
+
WHERE action = ? AND status = 'approved' AND (target_uri IS NULL OR target_uri = ? OR ? IS NULL)
|
|
392
|
+
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function qt(e,t,r,n){let i=r==="generated_write"&&t.approvals.generatedWritesRequireApproval,o=!i||In(e,r,n);return{action:r,target_uri:n??null,approval_required:i,approved:o,decision:o?"allow":"requires_approval"}}function ye(e,t){return`${e}_${Ln("sha256").update(t).digest("hex").slice(0,20)}`}function z(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function T(e){return typeof e==="string"&&e.length>0?e:void 0}function jn(e){let t=T(e.source_ref)??T(e.source_uri)??T(e.uri);if(t)return t;let r=T(e.file_id);if(r){let o=T(e.revision_id)??T(e.revision),c=`open-files://file/${encodeURIComponent(r)}`;return o?`${c}/revision/${encodeURIComponent(o)}`:c}let n=T(e.source_id),i=T(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Outbox event is missing source_ref, file_id, or source_id/path.")}function Kn(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Mn(e){return T(e.hash)??T(e.checksum)??T(e.sha256)??null}function Fn(e,t,r){return T(e.revision_id)??T(e.revision)??T(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??null}function Wn(e){return(T(e.event)??T(e.type)??T(e.action)??T(e.change_type)??"changed").toLowerCase()}function Xn(e){let t=T(e.path);return T(e.title)??T(e.name)??(t?Un(t):null)}function $n(e,t){let r=jn(e),n=U(r),i=Mn(e);return{raw:e,eventType:Wn(e),sourceRef:r,sourceUri:Kn(r,n),kind:n.kind,title:Xn(e),revision:Fn(e,n,i),hash:i,status:T(e.status)?.toLowerCase()??null,updatedAt:T(e.updated_at)??t,acl:e.permissions??e.acl??void 0}}function qn(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Outbox array parse failed.");return r.map((n)=>{let i=z(n);if(!i)throw Error("Outbox array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=z(r);if(!n)throw Error("Outbox object parse failed.");if(Array.isArray(n.events))return n.events.map((i)=>{let o=z(i);if(!o)throw Error("Outbox events entries must be objects.");return o});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let o=z(JSON.parse(i));if(!o)throw Error("Outbox JSONL entries must be objects.");return o})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=z(JSON.parse(r));if(!n)throw Error("Outbox JSONL entries must be objects.");return n})}async function Hn(e,t,r){let n=new URL(e),i=n.hostname,o=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!o)throw Error(`Invalid S3 outbox URI: ${e}`);if(r)X(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,s=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:o}));if(!s.Body)return"";return await s.Body.transformToString()}async function Bn(e,t,r){if(e.startsWith("s3://"))return Hn(e,t,r);if(!Dn(e))throw Error(`Outbox not found: ${e}`);return Pn(e,"utf8")}function Ht(e,t){let r={};if(e)try{r=z(JSON.parse(e))??{}}catch{r={}}return JSON.stringify({...r,...t})}function zn(e,t,r){let n=ye("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
393
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
394
|
+
ON CONFLICT(uri) DO UPDATE SET
|
|
395
|
+
kind = excluded.kind,
|
|
396
|
+
title = COALESCE(excluded.title, sources.title),
|
|
397
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let o={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)o.status=t.status;if(T(t.raw.path))o.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[Ht(i.metadata_json,o),t.acl===void 0?null:JSON.stringify(t.acl),t.acl===void 0?null:JSON.stringify(t.acl),t.updatedAt,i.id]),i.id}function Gn(e,t,r,n){if(!r.revision)return null;let i=ye("rev",`${t}\x00${r.revision}`),o={source_ref:r.sourceRef,source_uri:r.sourceUri,status:r.status,last_outbox_event:r.eventType,reindex_required:!0};return e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
398
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
399
|
+
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
400
|
+
hash = COALESCE(excluded.hash, source_revisions.hash),
|
|
401
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,T(r.raw.extracted_text_ref)??null,JSON.stringify(o),n]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision)?.id??null}function Jn(e,t,r){if(r.revision)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").all(t,r.revision).map((n)=>n.id);if(r.hash)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND hash = ?").all(t,r.hash).map((n)=>n.id);return e.query("SELECT id FROM source_revisions WHERE source_id = ?").all(t).map((n)=>n.id)}function Yn(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t),n=0,i=0;for(let c of r){let u=e.query("SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?").get(c.id);n+=u?.n??0;let _=e.query("SELECT COUNT(*) AS n FROM vector_index_entries WHERE chunk_id = ?").get(c.id);i+=_?.n??0,e.run("DELETE FROM vector_index_entries WHERE chunk_id = ?",[c.id]),e.run("DELETE FROM chunk_embeddings WHERE chunk_id = ?",[c.id]),e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[c.id])}e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]);let o=e.query("SELECT metadata_json FROM source_revisions WHERE id = ?").get(t);return e.run("UPDATE source_revisions SET metadata_json = ? WHERE id = ?",[Ht(o?.metadata_json,{reindex_required:!0,invalidated_at:new Date().toISOString()}),t]),{chunksDeleted:r.length,embeddingsDeleted:n,vectorEntriesDeleted:i}}function Vn(e,t){return t==="deleted"||["delete","deleted","remove","removed"].includes(e)}function Qn(e){return["move","moved","rename","renamed","path_changed"].includes(e)}function Zn(e){return["permission","permissions","permission_changed","acl_changed"].includes(e)}async function Bt(e){let t=(e.now??new Date).toISOString();if(e.safetyPolicy)$(e.dbPath,e.safetyPolicy);O(e.dbPath);let r=await Bn(e.input,e.config,e.safetyPolicy),n=qn(r),i=w(e.dbPath),o=`run_${Cn()}`;try{return i.transaction(()=>{i.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
402
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[o,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:n.length}),t,t]);let c=new Set,u=new Set,_=0,a=0,d=0,s=0,l=0,f=0,g=0;return A(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:n.length,read_only:!0},created_at:t}),n.forEach((k,h)=>{let m=$n(k,t),x=zn(i,m,t);c.add(x);let y=Gn(i,x,m,t);if(y)u.add(y);let p=Jn(i,x,m);for(let S of p){u.add(S);let R=Yn(i,S);_+=R.chunksDeleted,a+=R.embeddingsDeleted,d+=R.vectorEntriesDeleted,s+=1}if(Vn(m.eventType,m.status))l+=1;if(Qn(m.eventType))f+=1;if(Zn(m.eventType)||m.acl!==void 0)g+=1;i.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
403
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[ye("evt",`${o}\x00${h}\x00${m.sourceRef}\x00${m.eventType}`),o,"info",m.eventType,JSON.stringify({source_ref:m.sourceRef,source_uri:m.sourceUri,revision:m.revision,hash:m.hash,status:m.status,affected_revisions:p.length}),m.updatedAt])}),i.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
404
|
+
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[ye("usage",o),o,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),A(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:o,events:n.length,sources:c.size,revisions:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:o,events_seen:n.length,sources_touched:c.size,revisions_touched:u.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:d,stale_revisions:s,deleted_sources:l,moved_sources:f,permission_updates:g}})()}finally{i.close()}}import{createHash as ei}from"crypto";import{existsSync as ti,readFileSync as ri}from"fs";import{basename as ni}from"path";function Be(e,t){return`${e}_${ei("sha256").update(t).digest("hex").slice(0,20)}`}function G(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function v(e){return typeof e==="string"&&e.length>0?e:void 0}function ii(e){return typeof e==="number"&&Number.isFinite(e)?e:void 0}function si(e){let t=v(e.source_ref)??v(e.source_uri)??v(e.uri);if(t)return t;let r=v(e.file_id);if(r){let o=v(e.revision_id)??v(e.revision),c=`open-files://file/${encodeURIComponent(r)}`;return o?`${c}/revision/${encodeURIComponent(o)}`:c}let n=v(e.source_id),i=v(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Manifest item is missing source_ref, file_id, or source_id/path.")}function oi(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function ai(e){let t=v(e.extracted_text)??v(e.text)??v(e.content_text)??v(e.markdown);if(t!==void 0)return t;let r=e.content;return typeof r==="string"?r:null}function ci(e){let t=v(e.extracted_text_ref)??v(e.extracted_text_uri)??v(e.text_ref);if(t)return t;let r=G(e.content);return v(r?.extracted_text_ref)??v(r?.extracted_text_uri)??null}function di(e){let t=v(e.path);return v(e.title)??v(e.name)??(t?ni(t):null)}function ui(e){return v(e.hash)??v(e.checksum)??v(e.sha256)??null}function li(e,t,r){return v(e.revision_id)??v(e.revision)??v(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??v(e.updated_at)??"current"}function _i(e,t){let r={};for(let[n,i]of Object.entries(e)){if(["text","content","content_text","extracted_text","markdown"].includes(n))continue;r[n]=i}return r.source_ref=t.sourceRef,r.source_uri=t.sourceUri,r.status=t.status,r}function fi(e,t){let r=si(e),n=U(r),i=oi(r,n),o=ui(e),c=v(e.status)??"active";return{raw:e,sourceRef:r,sourceUri:i,kind:n.kind,title:di(e),revision:li(e,n,o),hash:o,extractedTextUri:ci(e),text:ai(e),metadata:_i(e,{sourceRef:r,sourceUri:i,status:c}),acl:e.permissions??e.acl??{},status:c,updatedAt:v(e.updated_at)??t}}function gi(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Manifest array parse failed.");return r.map((n)=>{let i=G(n);if(!i)throw Error("Manifest array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=G(r);if(!n)throw Error("Manifest object parse failed.");if(Array.isArray(n.items))return n.items.map((i)=>{let o=G(i);if(!o)throw Error("Manifest items entries must be objects.");return o});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let o=G(JSON.parse(i));if(!o)throw Error("Manifest JSONL entries must be objects.");return o})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=G(JSON.parse(r));if(!n)throw Error("Manifest JSONL entries must be objects.");return n})}async function pi(e,t,r){let n=new URL(e),i=n.hostname,o=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!o)throw Error(`Invalid S3 manifest URI: ${e}`);if(r)X(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,s=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:o}));if(!s.Body)return"";return await s.Body.transformToString()}async function hi(e,t,r){if(e.startsWith("s3://"))return pi(e,t,r);if(!ti(e))throw Error(`Manifest not found: ${e}`);return ri(e,"utf8")}function mi(e,t,r){let n=e.replace(/\r\n/g,`
|
|
405
|
+
`);if(!n.trim())return[];let i=[],o=0;while(o<n.length){let c=Math.min(n.length,o+t),u=c;if(c<n.length){let a=n.lastIndexOf(`
|
|
406
|
+
|
|
407
|
+
`,c),d=n.lastIndexOf(". ",c),s=Math.max(a,d);if(s>o+Math.floor(t*0.5))u=s+(s===a?2:1)}let _=n.slice(o,u).trim();if(_)i.push({ordinal:i.length,text:_,startOffset:o,endOffset:u});if(u>=n.length)break;o=Math.max(0,u-r)}return i}function ki(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Ei(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t);for(let n of r)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[n.id]);return e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]),r.length}function yi(e,t,r){let n=Be("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
408
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
409
|
+
ON CONFLICT(uri) DO UPDATE SET
|
|
410
|
+
kind = excluded.kind,
|
|
411
|
+
title = excluded.title,
|
|
412
|
+
metadata_json = excluded.metadata_json,
|
|
413
|
+
acl_json = excluded.acl_json,
|
|
414
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function bi(e,t,r,n){let i=Be("rev",`${t}\x00${r.revision}`);e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
415
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
416
|
+
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
417
|
+
hash = excluded.hash,
|
|
418
|
+
extracted_text_uri = excluded.extracted_text_uri,
|
|
419
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let o=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision);if(!o)throw Error(`Failed to upsert source revision: ${r.sourceRef}`);return o.id}function vi(e,t,r,n,i,o,c){if(!r.text||r.status.toLowerCase()==="deleted")return{chunksInserted:0,redactions:0};let u=ke(r.text,c);if(u.findings.length>0)Ee(e,{source_uri:r.sourceUri,findings:u.findings,metadata:{source_ref:r.sourceRef,revision:r.revision},created_at:n}),A(e,{event_type:"redaction",action:"source_text_redact",target_uri:r.sourceUri,decision:"redacted",metadata:{findings:u.findings.length,source_ref:r.sourceRef,revision:r.revision},created_at:n});let _=mi(u.text,i,o);for(let a of _){let d=Be("chk",`${t}\x00${a.ordinal}\x00${a.text}`),s=W({source_ref:r.sourceRef,source_uri:r.sourceUri,source_kind:r.kind,source_revision_id:t,revision:r.revision,hash:r.hash,chunk_id:d,start_offset:a.startOffset,end_offset:a.endOffset,status:r.status,resolver:"open-files-read-only"}),l=mt({source_ref:r.sourceRef,source_uri:r.sourceUri,source_kind:r.kind,source_revision_id:t,revision:r.revision,hash:r.hash,status:r.status,path:v(r.raw.path)??null,mime:v(r.raw.mime)??v(r.raw.content_type)??null,size:ii(r.raw.size)??null},s);e.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
420
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[d,t,"source",a.ordinal,a.text,ki(a.text),a.startOffset,a.endOffset,JSON.stringify(l),n]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[d,a.text,r.title??"",r.sourceUri])}return{chunksInserted:_.length,redactions:u.findings.length}}async function zt(e){let t=e.now??new Date;if(e.safetyPolicy)$(e.dbPath,e.safetyPolicy);O(e.dbPath);let r=await hi(e.input,e.config,e.safetyPolicy),n=gi(r);return ze({dbPath:e.dbPath,items:n,sourceLabel:e.input,safetyPolicy:e.safetyPolicy,now:t,maxChunkChars:e.maxChunkChars,chunkOverlapChars:e.chunkOverlapChars})}async function ze(e){let t=(e.now??new Date).toISOString(),r=e.maxChunkChars??4000,n=e.chunkOverlapChars??200;if(r<500)throw Error("maxChunkChars must be at least 500.");if(n<0||n>=r)throw Error("chunkOverlapChars must be less than maxChunkChars.");if(e.safetyPolicy)$(e.dbPath,e.safetyPolicy);O(e.dbPath);let i=w(e.dbPath);try{return i.transaction(()=>{let c=new Set,u=new Set,_=0,a=0,d=0,s=0;A(i,{event_type:"source_read",action:e.readAction??(e.sourceLabel.startsWith("s3://")?"s3_manifest_read":"local_manifest_read"),target_uri:e.sourceLabel,decision:"allow",metadata:{items:e.items.length,read_only:!0},created_at:t});for(let l of e.items){let f=fi(l,t),g=yi(i,f,t),k=bi(i,g,f,t);if(c.add(g),u.add(k),f.text||f.status.toLowerCase()==="deleted")a+=Ei(i,k);let h=vi(i,k,f,t,r,n,e.safetyPolicy);_+=h.chunksInserted,d+=h.redactions}return A(i,{event_type:"write",action:"knowledge_manifest_ingest",target_uri:e.dbPath,decision:"allow",metadata:{items:e.items.length,sources:c.size,revisions:u.size,chunks_inserted:_,redactions:d},created_at:t}),{path:e.sourceLabel,db_path:e.dbPath,items_seen:e.items.length,sources_upserted:c.size,revisions_upserted:u.size,chunks_inserted:_,chunks_deleted:a,redactions:d,skipped:s}})()}finally{i.close()}}import{createHash as Ni}from"crypto";import{existsSync as Ai,readFileSync as Ii}from"fs";import{basename as we}from"path";function be(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function q(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function Gt(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function wi(e,t){let r=e.mode;if(typeof r==="string"&&r!=="read_only")throw Error(`Source resolver denied ${t}. Permission mode is ${r}, expected read_only.`);let n=e.denied_purposes;if(Array.isArray(n)&&n.includes(t))throw Error(`Source resolver denied ${t}. Purpose is explicitly denied.`);let i=e.allowed_purposes;if(Array.isArray(i)&&i.length>0&&!i.includes(t))throw Error(`Source resolver denied ${t}. Allowed purposes: ${i.join(", ")}`)}function Ti(e,t,r){if(!t)return r;try{let n=U(e);if(n.kind==="open-files"&&n.entity==="file")return`${e}/revision/${encodeURIComponent(t.revision)}`}catch{return r}return r}function xi(e,t,r){return e.query(`SELECT id, uri, kind, title, metadata_json, acl_json, updated_at
|
|
421
|
+
FROM sources
|
|
422
|
+
WHERE uri = ? OR uri = ?
|
|
423
|
+
ORDER BY CASE WHEN uri = ? THEN 0 ELSE 1 END
|
|
424
|
+
LIMIT 1`).get(t,r,t)??null}function Si(e,t,r){if(r)return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
425
|
+
FROM source_revisions
|
|
426
|
+
WHERE source_id = ? AND revision = ?
|
|
427
|
+
LIMIT 1`).get(t,r)??null;return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
428
|
+
FROM source_revisions
|
|
429
|
+
WHERE source_id = ?
|
|
430
|
+
ORDER BY created_at DESC, revision DESC
|
|
431
|
+
LIMIT 1`).get(t)??null}function Ri(e,t){if(!t)return 0;return e.query("SELECT COUNT(*) AS n FROM chunks WHERE source_revision_id = ?").get(t)?.n??0}function Oi(e,t,r){if(!t||r<=0)return[];return e.query(`SELECT id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json
|
|
432
|
+
FROM chunks
|
|
433
|
+
WHERE source_revision_id = ?
|
|
434
|
+
ORDER BY ordinal ASC
|
|
435
|
+
LIMIT ?`).all(t,r)}async function ve(e){let t=e.purpose??"knowledge_answer",r=Math.max(0,Math.min(e.limit??10,100)),n=(e.now??new Date).toISOString(),i=U(e.sourceRef),o=Kt(e.sourceRef,i),c=Mt(e.sourceRef);if(e.safetyPolicy){if(!e.safetyPolicy.readOnlySourceAccess)throw Error("Safety policy denied source resolution.");$(e.dbPath,e.safetyPolicy)}O(e.dbPath);let u=w(e.dbPath);try{return u.transaction(()=>{let _=xi(u,o,e.sourceRef);if(!_)return A(u,{event_type:"source_read",action:"open_files_resolve_missing",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:o},created_at:n}),{source_ref:e.sourceRef,source_uri:o,purpose:t,read_only:!0,resolved:!1,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:null,revision:null,content:{mime:null,size:null,hash:null,text_available:!1,chunks_total:0,chunks_returned:0,char_count_returned:0,extracted_text_ref:null,bytes_available:!1,bytes_exposed:!1},chunks:[],citations:[]};let a=be(_.metadata_json),d=be(_.acl_json);try{wi(d,t)}catch(p){throw A(u,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"deny",metadata:{purpose:t,read_only:!0,source_uri:_.uri,error:p instanceof Error?p.message:String(p)},created_at:n}),p}let s=Si(u,_.id,c),l=be(s?.metadata_json),f=Ri(u,s?.id??null),g=Oi(u,s?.id??null,r),k=Ti(_.uri,s,e.sourceRef),h=g.map((p)=>{let S=be(p.metadata_json),R={resolver:"open-files-read-only",mode:"local_catalog",purpose:t,read_only:!0,source_ref:q(S,["source_ref"])??k,source_uri:_.uri,source_revision_id:s?.id??null,revision:s?.revision??null,hash:s?.hash??q(S,["hash"]),chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,resolved_at:n},b=W({source_ref:R.source_ref,source_uri:R.source_uri,source_kind:_.kind,source_revision_id:R.source_revision_id,revision:R.revision,hash:R.hash,chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,status:q(S,["status"]),resolver:R.resolver});return{id:p.id,kind:p.kind,ordinal:p.ordinal,text:p.text,token_count:p.token_count,start_offset:p.start_offset,end_offset:p.end_offset,metadata:S,evidence:R,provenance:b}}),m=h.map((p)=>({source_ref:p.evidence.source_ref,source_uri:_.uri,chunk_id:p.id,quote:p.text.slice(0,500),start_offset:p.start_offset,end_offset:p.end_offset,evidence:p.evidence,provenance:p.provenance}));A(u,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:_.uri,revision:s?.revision??null,chunks_returned:h.length,chunks_total:f},created_at:n});let x=q(a,["mime","content_type"])??q(l,["mime","content_type"]),y=Gt(a,["size","size_bytes"])??Gt(l,["size","size_bytes"]);return{source_ref:k,source_uri:_.uri,purpose:t,read_only:!0,resolved:!0,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:{id:_.id,uri:_.uri,kind:_.kind,title:_.title,metadata:a,permissions:d,updated_at:_.updated_at},revision:s?{id:s.id,revision:s.revision,hash:s.hash,extracted_text_uri:s.extracted_text_uri,metadata:l,created_at:s.created_at,reindex_required:l.reindex_required===!0}:null,content:{mime:x,size:y,hash:s?.hash??q(a,["hash","checksum","sha256"]),text_available:f>0,chunks_total:f,chunks_returned:h.length,char_count_returned:h.reduce((p,S)=>p+S.text.length,0),extracted_text_ref:s?.extracted_text_uri??q(l,["extracted_text_ref","extracted_text_uri"]),bytes_available:!1,bytes_exposed:!1},chunks:h,citations:m}})()}finally{u.close()}}function J(e){return`sha256:${Ni("sha256").update(e).digest("hex")}`}function Li(e){return e.replace(/<script[\s\S]*?<\/script>/gi," ").replace(/<style[\s\S]*?<\/style>/gi," ").replace(/<[^>]+>/g," ").replace(/ /g," ").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/\s+\n/g,`
|
|
436
|
+
`).replace(/\n\s+/g,`
|
|
437
|
+
`).replace(/[ \t]{2,}/g," ").trim()}async function Ci(e,t,r){let n=new URL(e),i=n.hostname,o=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!o)throw Error(`Invalid S3 source URI: ${e}`);if(r)X(e,r);let[{S3Client:c,GetObjectCommand:u},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,s=await new c({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new u({Bucket:i,Key:o}));if(!s.Body)return"";return await s.Body.transformToString()}async function Di(e,t){if(t)me(t);let r=await fetch(e,{headers:{accept:"text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5","user-agent":"@hasna/knowledge source-ingest"}});if(!r.ok)throw Error(`Web source read failed ${r.status}: ${e}`);let n=r.headers.get("content-type"),i=await r.text();return{text:n?.includes("html")?Li(i):i,mime:n}}function Te(e){if(e.kind==="file")return we(e.path);if(e.kind==="s3")return we(e.key);if(e.kind==="web")return we(new URL(e.url).pathname)||e.url;return e.path?we(e.path):e.id}async function Jt(e,t,r){if(e.kind==="file"){if(!Ai(e.path))throw Error(`Source file not found: ${e.path}`);let n=Ii(e.path,"utf8");return{text:n,contentSource:"file",title:Te(e),mime:"text/plain",size:n.length,hash:J(n),revision:null,extractedTextRef:null,metadata:{path:e.path},permissions:{mode:"read_only"}}}if(e.kind==="s3"){let n=await Ci(e.uri,t,r);return{text:n,contentSource:"s3",title:Te(e),mime:"text/plain",size:n.length,hash:J(n),revision:null,extractedTextRef:null,metadata:{bucket:e.bucket,key:e.key},permissions:{mode:"read_only"}}}if(e.kind==="web"){let n=await Di(e.url,r);return{text:n.text,contentSource:"web",title:Te(e),mime:n.mime,size:n.text.length,hash:J(n.text),revision:null,extractedTextRef:null,metadata:{url:e.url},permissions:{mode:"read_only"}}}throw Error(`Direct source reading is not available for ${e.uri}`)}async function Pi(e,t,r){if(e.startsWith("open-files://"))throw Error("Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.");let n=U(e);return{text:(await Jt(n,t,r)).text,contentSource:"extracted_text_ref"}}async function Ui(e){let t=await ve({dbPath:e.dbPath,sourceRef:e.sourceRef,purpose:e.purpose??"knowledge_index",limit:100,safetyPolicy:e.safetyPolicy,now:e.now});if(!t.resolved)throw Error("Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.");if(t.revision?.extracted_text_uri&&!t.content.text_available){let n=await Pi(t.revision.extracted_text_uri,e.config,e.safetyPolicy);return{text:n.text,contentSource:n.contentSource,title:t.source?.title??null,mime:t.content.mime,size:n.text.length,hash:t.revision.hash??J(n.text),revision:t.revision.revision,extractedTextRef:t.revision.extracted_text_uri,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}if(t.chunks.length===0)throw Error("Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.");let r=t.chunks.map((n)=>n.text).join(`
|
|
438
|
+
|
|
439
|
+
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??J(r),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function ji(e,t,r,n){let i=r.hash??J(r.text),o={...r.metadata,source_ref:e,content_source:r.contentSource,read_only:!0},c={source_ref:e,name:r.title??Te(t),mime:r.mime??"text/plain",size:r.size??r.text.length,hash:i,revision:r.revision??i,status:"active",updated_at:new Date().toISOString(),permissions:{mode:"read_only",allowed_purposes:[n],...r.permissions},metadata:o,extracted_text_ref:r.extractedTextRef,extracted_text:r.text};if(t.kind==="open-files"){if(t.entity==="file")c.file_id=t.id;if(t.entity==="source")c.source_id=t.id,c.path=t.path}if(t.kind==="file")c.path=t.path;if(t.kind==="s3")c.path=t.key;if(t.kind==="web")c.url=t.url;return c}async function Yt(e){let t=e.purpose??"knowledge_index",r=U(e.sourceRef),n=r.kind==="open-files"?await Ui(e):await Jt(r,e.config,e.safetyPolicy),i=ji(e.sourceRef,r,n,t);return{...await ze({dbPath:e.dbPath,items:[i],sourceLabel:e.sourceRef,readAction:"source_ref_ingest_read",safetyPolicy:e.safetyPolicy,now:e.now}),source_ref:e.sourceRef,content_source:n.contentSource,read_only:!0,hash:String(i.hash)}}import{createHash as Ki,randomUUID as Mi}from"crypto";var Vt=[{kind:"schema",prefix:"schemas/",description:"Machine-readable agent schemas and source rules."},{kind:"index",prefix:"indexes/",description:"Small orientation indexes and future shard manifests."},{kind:"log",prefix:"logs/",description:"Append-only JSONL run and wiki-maintenance log partitions."},{kind:"run",prefix:"runs/",description:"Prompt/tool/cost ledgers and generated output records."},{kind:"wiki_page",prefix:"wiki/",description:"Generated cited Markdown pages, not raw source files."},{kind:"export",prefix:"exports/",description:"Portable exports and snapshots of derived knowledge state."}];function Qt(e){let t=typeof e==="string"?Buffer.from(e):Buffer.from(e);return{hash:`sha256:${Ki("sha256").update(t).digest("hex")}`,size_bytes:t.byteLength}}function Zt(e){return Vt.find((r)=>e.startsWith(r.prefix))?.kind??"artifact"}function er(e,t,r="global"){let n=Ge(e,t),i=e.storage.s3??null,o=i?.prefix?.replace(/^\/+|\/+$/g,"")??"",c=i?`s3://${i.bucket}/${o?`${o}/`:""}`:"";return{scope:r,mode:e.mode,storage_type:e.storage.type,workspace_home:t.home,local_layout:{app_path:V,config_path:t.configPath,json_store_path:t.jsonStorePath,knowledge_db_path:t.knowledgeDbPath,directories:{artifacts:t.artifactsDir,cache:t.cacheDir,exports:t.exportsDir,indexes:t.indexesDir,logs:t.logsDir,runs:t.runsDir,schemas:t.schemasDir,wiki:t.wikiDir}},artifact_store:{type:e.storage.type,artifacts_root:e.storage.artifacts_root,uri_prefix:e.storage.type==="s3"?c:`file://${t.artifactsDir}/`,s3:i?{bucket:i.bucket,prefix:o,region:i.region??null,profile:i.profile??null,server_side_encryption:i.server_side_encryption??null,kms_key_configured:Boolean(i.kms_key_id)}:null},source_ownership:{owner:"open-files",preferred_ref:e.sources.preferred_ref,allowed_schemes:e.sources.allowed_schemes,raw_source_bytes_stored_in_open_knowledge:!1,stores:["source refs","source revisions and hashes","citation spans","redacted extracted chunks","embeddings","generated wiki artifacts","indexes","run ledgers"],does_not_store:["raw open-files bytes","S3 object credentials","connector secrets","hosted tenant ownership state"]},generated_artifacts:Vt,scalability:{catalog:"knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.",indexes:"Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.",logs:"Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.",markdown:"Markdown pages are the readable wiki layer over DB/object-store state."},warnings:n.warnings}}function Ge(e,t){let r=[],n=[];if(!t.home.endsWith(V))n.push(`Workspace home does not end with ${V}: ${t.home}`);if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)r.push("storage.s3.bucket is required when storage.type is s3.");if(!e.storage.s3?.prefix)n.push("storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.");if(e.mode==="local")n.push("storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.")}if(e.storage.type==="local"&&e.storage.s3)n.push("storage.s3 is configured but ignored while storage.type is local.");if(e.sources.preferred_ref!=="open-files")n.push("sources.preferred_ref should stay open-files for durable company knowledge.");if(!e.sources.allowed_schemes.includes("open-files"))r.push("sources.allowed_schemes must include open-files.");return{ok:r.length===0,errors:r,warnings:n}}function tr(e,t,r=new Date){let n=r.toISOString(),i=e.prepare(`
|
|
418
440
|
INSERT INTO storage_objects (
|
|
419
441
|
id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
|
|
420
442
|
)
|
|
@@ -426,7 +448,7 @@ VALUES (4, datetime('now'));
|
|
|
426
448
|
size_bytes = excluded.size_bytes,
|
|
427
449
|
metadata_json = excluded.metadata_json,
|
|
428
450
|
updated_at = excluded.updated_at
|
|
429
|
-
`);e.transaction((
|
|
451
|
+
`);e.transaction((c)=>{for(let u of c)i.run(Mi(),u.uri,u.kind,u.content_type??null,u.hash??null,u.size_bytes??null,JSON.stringify({key:u.key,...u.metadata??{}}),n,n)})(t)}import{createHash as Fi}from"crypto";function Wi(e){let t=String(e.getUTCFullYear()),r=String(e.getUTCMonth()+1).padStart(2,"0"),n=String(e.getUTCDate()).padStart(2,"0");return{year:t,month:r,day:n}}function Je(e,t){return`${e}_${Fi("sha256").update(t).digest("hex").slice(0,20)}`}function Xi(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function $i(){return`# Knowledge Agent Schema v1
|
|
430
452
|
|
|
431
453
|
## Source Rules
|
|
432
454
|
|
|
@@ -451,7 +473,7 @@ VALUES (4, datetime('now'));
|
|
|
451
473
|
## Lint Rules
|
|
452
474
|
|
|
453
475
|
- Flag stale pages, missing citations, contradictions, orphan pages, duplicate pages, and unresolved source refs.
|
|
454
|
-
`}function
|
|
476
|
+
`}function qi(){return`# Knowledge Index
|
|
455
477
|
|
|
456
478
|
This is a compact orientation index for agents. It is not the full search index.
|
|
457
479
|
|
|
@@ -466,19 +488,19 @@ This is a compact orientation index for agents. It is not the full search index.
|
|
|
466
488
|
|
|
467
489
|
Raw source files are resolved through open-files. This app stores source refs,
|
|
468
490
|
citations, chunks, generated wiki artifacts, indexes, and run records.
|
|
469
|
-
`}function
|
|
491
|
+
`}function rr(){return`# Wiki
|
|
470
492
|
|
|
471
493
|
Generated durable knowledge pages live here.
|
|
472
494
|
|
|
473
495
|
Pages should be concise, cited, and organized for both humans and agents.
|
|
474
|
-
`}async function
|
|
475
|
-
`,content_type:"application/x-ndjson"}],
|
|
476
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
496
|
+
`}async function nr(e,t=new Date){let{year:r,month:n,day:i}=Wi(t),o="schemas/v1.md",c="indexes/root.md",u="wiki/README.md",_=`logs/${r}/${n}/${i}.jsonl`,a={ts:t.toISOString(),event:"wiki_layout_initialized",schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md"},d=[{key:"schemas/v1.md",body:$i(),content_type:"text/markdown"},{key:"indexes/root.md",body:qi(),content_type:"text/markdown"},{key:"wiki/README.md",body:rr(),content_type:"text/markdown"},{key:_,body:`${JSON.stringify(a)}
|
|
497
|
+
`,content_type:"application/x-ndjson"}],s=await Promise.all(d.map(async(l)=>{let f=await e.put(l);return{key:f.key,uri:f.uri,kind:Zt(l.key),content_type:l.content_type,metadata:{provenance:je({generated_from:"wiki_layout_init",artifact_key:l.key,citation_required:l.key.startsWith("wiki/")||l.key.startsWith("indexes/")})},...Qt(l.body)}}));return{schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md",log_key:_,artifacts:s,written:["schemas/v1.md","indexes/root.md","wiki/README.md",_]}}function Ye(e){let t=e.metadata?.provenance;if(t&&typeof t==="object"&&!Array.isArray(t))return t;return je({generated_from:"wiki_layout_init",artifact_key:e.key})}function Hi(e,t,r,n,i,o){let c=Ye(n),u=Je("chk",`${t}\x00${n.hash??n.uri}`),_=e.query("SELECT id FROM chunks WHERE wiki_page_id = ?").all(t);for(let a of _)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[a.id]);e.run("DELETE FROM chunks WHERE wiki_page_id = ?",[t]),e.run(`INSERT INTO chunks (id, wiki_page_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
498
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[u,t,"wiki",0,i,Xi(i),0,i.length,JSON.stringify({artifact_key:n.key,artifact_uri:n.uri,content_hash:n.hash??null,provenance:c}),o]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[u,i,r,n.uri])}function ir(e,t,r=new Date){let n=r.toISOString(),i=t.find((c)=>c.key.endsWith("indexes/root.md")),o=t.find((c)=>c.key.endsWith("wiki/README.md"));if(i)e.run(`INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
|
|
477
499
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
478
500
|
ON CONFLICT(kind, name, shard_key) DO UPDATE SET
|
|
479
501
|
artifact_uri = excluded.artifact_uri,
|
|
480
502
|
metadata_json = excluded.metadata_json,
|
|
481
|
-
updated_at = excluded.updated_at`,[
|
|
503
|
+
updated_at = excluded.updated_at`,[Je("idx","root:indexes/root.md"),"root","root",i.uri,"root",JSON.stringify({artifact_key:i.key,content_hash:i.hash??null,provenance:Ye(i)}),n,n]);if(o){let c=Je("wiki","wiki/README.md");e.run(`INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
|
|
482
504
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
483
505
|
ON CONFLICT(path) DO UPDATE SET
|
|
484
506
|
title = excluded.title,
|
|
@@ -486,7 +508,7 @@ Pages should be concise, cited, and organized for both humans and agents.
|
|
|
486
508
|
content_hash = excluded.content_hash,
|
|
487
509
|
status = excluded.status,
|
|
488
510
|
metadata_json = excluded.metadata_json,
|
|
489
|
-
updated_at = excluded.updated_at`,[
|
|
511
|
+
updated_at = excluded.updated_at`,[c,"wiki/README.md","Wiki",o.uri,o.hash??null,"active",JSON.stringify({artifact_key:o.key,provenance:Ye(o)}),n,n]),Hi(e,c,"Wiki",o,rr(),n)}}class sr{options;ensuredWorkspace;cachedConfig;constructor(e={}){this.options=e}get scope(){return this.options.scope??"global"}get workspace(){return this.ensuredWorkspace??Ze(this.options.scope,this.options.cwd)}ensureWorkspace(){if(!this.ensuredWorkspace)this.ensuredWorkspace=Qe(this.workspace.home);return this.ensuredWorkspace}jsonStorePath(){return this.ensureWorkspace().jsonStorePath}config(){if(!this.cachedConfig){let e=this.ensureWorkspace();this.cachedConfig=et(e.configPath)}return this.cachedConfig}safetyPolicy(){return Xt(this.config(),this.ensureWorkspace())}artifactStore(){return ct(this.config(),this.ensureWorkspace())}storageContract(){return er(this.config(),this.ensureWorkspace(),this.scope)}validateStorage(){return Ge(this.config(),this.ensureWorkspace())}paths(){let e=this.ensureWorkspace();return{ok:!0,scope:this.scope,home:e.home,config_path:e.configPath,json_store_path:e.jsonStorePath,knowledge_db_path:e.knowledgeDbPath,artifacts_dir:e.artifactsDir,indexes_dir:e.indexesDir,logs_dir:e.logsDir,runs_dir:e.runsDir,schemas_dir:e.schemasDir,wiki_dir:e.wikiDir,config:this.config(),message:e.home}}initDb(){return O(this.ensureWorkspace().knowledgeDbPath)}dbStats(){let e=this.ensureWorkspace();return O(e.knowledgeDbPath),it(e.knowledgeDbPath)}async initWiki(){let e=this.ensureWorkspace();O(e.knowledgeDbPath);let t=await nr(this.artifactStore()),r=w(e.knowledgeDbPath);try{tr(r,t.artifacts),ir(r,t.artifacts)}finally{r.close()}return t}async ingestManifest(e){let t=this.ensureWorkspace();return zt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}async ingestSource(e,t){let r=this.ensureWorkspace();return Yt({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t,config:this.config(),safetyPolicy:this.safetyPolicy()})}async resolveSource(e,t={}){let r=this.ensureWorkspace();return ve({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t.purpose,limit:t.limit,safetyPolicy:this.safetyPolicy()})}async consumeOutbox(e){let t=this.ensureWorkspace();return Bt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}providerStatus(e=process.env){return ft(this.config(),e)}modelRegistry(){return Pe(this.config())}embeddingStatus(){let e=this.ensureWorkspace();return Tt(e.knowledgeDbPath)}async indexEmbeddings(e={}){let t=this.ensureWorkspace();return wt({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async semanticSearch(e){let t=this.ensureWorkspace();return ue({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async search(e){let t=this.ensureWorkspace();return ge({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async retrieveContext(e){let t=this.ensureWorkspace();return he({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async runPrompt(e){let t=this.ensureWorkspace();return Ut({...e,dbPath:t.knowledgeDbPath,config:this.config()})}}function or(e={}){return new sr(e)}import{basename as zi}from"path";var re={name:"@hasna/knowledge",version:"0.2.17",description:"Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",type:"module",bin:{knowledge:"bin/open-knowledge.js","open-knowledge":"bin/open-knowledge.js","open-knowledge-mcp":"bin/open-knowledge-mcp.js"},files:["bin","src","docs","LICENSE","README.md"],scripts:{test:"bun test","test:cli":"bun test tests/cli.test.ts",build:"bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",prepublishOnly:"bun run build",postinstall:"bun run build"},keywords:["knowledge","cli","agents","json","notes","local","store"],license:"Apache-2.0",publishConfig:{registry:"https://registry.npmjs.org",access:"public"},repository:{type:"git",url:"git+https://github.com/hasna/knowledge.git"},bugs:{url:"https://github.com/hasna/knowledge/issues"},author:"Hasna Inc. <hasna@example.com>",engines:{bun:">=1.0",node:">=18"},dependencies:{"@aws-sdk/client-s3":"^3.1063.0","@aws-sdk/credential-providers":"^3.1063.0","@ai-sdk/anthropic":"^3.0.81","@ai-sdk/deepseek":"^2.0.35","@ai-sdk/openai":"^3.0.68","@modelcontextprotocol/sdk":"^1.29.0",ai:"^6.0.197",zod:"^4.3.6"},devDependencies:{"@types/bun":"^1.3.14"}};var ar={debug:0,info:1,warn:2,error:3},Gi=()=>{if(process.env.DEBUG)return"debug";if(process.env.LOG_LEVEL==="debug")return"debug";if(process.env.LOG_LEVEL==="warn")return"warn";if(process.env.LOG_LEVEL==="error")return"error";return"info"};function H(e,t,r){if(ar[e]<ar[Gi()])return;let n={debug:"[DEBUG]",info:"[INFO]",warn:"[WARN]",error:"[ERROR]"}[e],i=r?`${n} ${t} ${JSON.stringify(r)}`:`${n} ${t}`;if(e==="error")console.error(i);else console.error(i)}var cr=["add","list","get","delete","update","archive","restore","upsert","untag","export","prune","dedupe","stats","paths","storage","db","wiki","source","ingest","reindex","search","ask","build","embeddings","providers","safety","help"],dr={ls:"list",rm:"delete",edit:"update",unarchive:"restore",knowledge:"ask"};function Ji(e){let t=[],r={};for(let n=0;n<e.length;n+=1){let i=e[n];if(!i.startsWith("-")){t.push(i);continue}switch(i){case"--json":r.json=!0;break;case"--yes":case"-y":r.yes=!0;break;case"--help":case"-h":r.help=!0;break;case"--version":case"-v":r.version=!0;break;case"--desc":r.desc=!0;break;case"--page":case"-p":r.page=Number(e[n+1]),n+=1;break;case"--limit":case"-l":r.limit=Number(e[n+1]),n+=1;break;case"--search":case"-s":r.search=e[n+1],n+=1;break;case"--sort":r.sort=e[n+1],n+=1;break;case"--id":r.id=e[n+1],n+=1;break;case"--store":r.store=e[n+1],n+=1;break;case"--title":r.title=e[n+1],n+=1;break;case"--content":r.content=e[n+1],n+=1;break;case"--url":r.url=e[n+1],n+=1;break;case"--tag":case"-t":r.tag=e[n+1],n+=1;break;case"--format":r.format=e[n+1],n+=1;break;case"--completions":r.completions=e[n+1],n+=1;break;case"--purpose":r.purpose=e[n+1],n+=1;break;case"--model":r.model=e[n+1],n+=1;break;case"--dimensions":r.dimensions=Number(e[n+1]),n+=1;break;case"--semantic":r.semantic=!0;break;case"--context":r.context=!0;break;case"--generate":r.generate=!0;break;case"--approve-write":r.approveWrite=!0;break;case"--fake":r.fake=!0;break;case"--no-color":r.noColor=!0;break;case"--scope":r.scope=e[n+1],n+=1;break;case"--older-than":r.olderThan=Number(e[n+1]),n+=1;break;case"--empty":r.empty=!0;break;case"--archived":r.archived=!0;break;case"--include-archived":r.includeArchived=!0;break;default:throw Error(`Unknown flag: ${i}. Run 'open-knowledge --help' for valid options.`)}}return{positional:t,flags:r}}function Yi(e){if(!e)return"";return dr[e]??e}function Vi(e,t){let r=Array.from({length:e.length+1},()=>Array(t.length+1).fill(0));for(let n=0;n<=e.length;n+=1)r[n][0]=n;for(let n=0;n<=t.length;n+=1)r[0][n]=n;for(let n=1;n<=e.length;n+=1)for(let i=1;i<=t.length;i+=1){let o=e[n-1]===t[i-1]?0:1;r[n][i]=Math.min(r[n-1][i]+1,r[n][i-1]+1,r[n-1][i-1]+o)}return r[e.length][t.length]}function Qi(e){if(!e)return"";let t=[...cr,...Object.keys(dr)],r="",n=Number.POSITIVE_INFINITY;for(let i of t){let o=Vi(e,i);if(o<n)n=o,r=i}return n<=3?r:""}function Zi(){return zi(process.argv[1]??"")==="knowledge"}function es(){console.log(`open-knowledge - local agent knowledge store
|
|
490
512
|
|
|
491
513
|
Usage:
|
|
492
514
|
open-knowledge <command> [options]
|
|
@@ -513,7 +535,8 @@ Commands:
|
|
|
513
535
|
ingest manifest <file|s3://> Ingest an open-files manifest into knowledge.db
|
|
514
536
|
ingest source <source-ref> Ingest a read-only source ref into knowledge.db
|
|
515
537
|
reindex outbox <file|s3://> Consume open-files change events and invalidate chunks
|
|
516
|
-
search <query> Hybrid search sources, wiki pages,
|
|
538
|
+
search <query> Hybrid search sources, wiki pages, indexes, or context
|
|
539
|
+
ask|build <prompt> Build a read-only citation answer/context pack
|
|
517
540
|
embeddings status|index|search Build/query local vector embeddings
|
|
518
541
|
providers status|models|check Inspect AI SDK provider config and credentials
|
|
519
542
|
safety status|check|approve|audit|redact
|
|
@@ -526,6 +549,9 @@ Global Options:
|
|
|
526
549
|
--model <provider:model> AI/embedding model ref
|
|
527
550
|
--dimensions <n> Embedding dimensions for local/fake providers
|
|
528
551
|
--semantic Include vector semantic results in search
|
|
552
|
+
--context Return a reranked citation context pack for search
|
|
553
|
+
--generate Call AI SDK text generation for ask/build
|
|
554
|
+
--approve-write Record approval intent for future durable wiki writes
|
|
529
555
|
--fake Use deterministic fake embeddings for local tests
|
|
530
556
|
--scope local|global|project Store scope (default: global ~/.hasna/apps/knowledge/)
|
|
531
557
|
--no-color Disable color output
|
|
@@ -563,5 +589,5 @@ Export Options:
|
|
|
563
589
|
|
|
564
590
|
Prune Options:
|
|
565
591
|
--older-than <days> Remove items older than N days
|
|
566
|
-
--empty Remove items with empty content`)}function
|
|
567
|
-
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search embeddings providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let r=_i(t[0]);if(!r||n.help||r==="help"){hi(t[1]);return}let i=Xt({scope:n.scope}),s=n.store;if(!s)if(n.scope==="project"||n.scope==="local")s=i.jsonStorePath();else s=ve();if(r==="paths"){E(i.paths(),n.json);return}if(r==="storage"){let a=t[1]??"status";if(a==="status"){let o=i.storageContract(),c=i.validateStorage();E({ok:c.ok,...o,validation:c,message:`${o.storage_type} artifact storage at ${o.artifact_store.uri_prefix}`},n.json);return}if(a==="validate"){let o=i.validateStorage();E({ok:o.ok,validation:o,message:o.ok?"Storage contract valid":`Storage contract invalid: ${o.errors.join("; ")}`},n.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(r==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let c=i.initDb();E({ok:!0,...c,message:`Initialized ${c.path}`},n.json);return}let o=i.dbStats();E({ok:!0,path:i.workspace.knowledgeDbPath,...o,message:`knowledge.db schema v${o.schema_version}`},n.json);return}if(r==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let o=await i.initWiki();E({ok:!0,...o,message:`Initialized wiki layout in ${i.workspace.home}`},n.json);return}if(r==="safety"){let a=t[1]??"status",o=i.ensureWorkspace(),c=i.safetyPolicy();i.initDb();let u=S(o.knowledgeDbPath);try{if(a==="status"){E({ok:!0,mode:c.mode,workspace:o.home,allow_write_roots:c.allowWriteRoots,read_only_source_access:c.readOnlySourceAccess,network:c.network,redaction:c.redaction,approvals:c.approvals,message:`Safety policy: ${c.mode}`},n.json);return}if(a==="check"){let _=t[2]??"generated_write",f=t[3]??null,m;try{if(_==="web_search")ue(c),m={action:_,target_uri:f,approval_required:!1,approved:!0,decision:"allow"};else if(_==="s3_read"){if(!f)throw Error("safety check s3_read requires an s3:// target.");F(f,c),m={action:_,target_uri:f,approval_required:!1,approved:!0,decision:"allow"}}else m=yt(u,c,_,f);R(u,{event_type:"safety_check",action:_,target_uri:f,decision:m.decision==="allow"?"allow":"requires_approval",metadata:m}),E({ok:!0,...m,message:`Safety check ${m.decision}`},n.json);return}catch(y){throw R(u,{event_type:"safety_check",action:_,target_uri:f,decision:"deny",metadata:{error:y instanceof Error?y.message:String(y)}}),y}}if(a==="approve"){let _=t[2]??"generated_write",f=t[3]??null,m=kt(u,{action:_,target_uri:f,reason:"local-cli approval",metadata:{scope:n.scope??"global"}});R(u,{event_type:"approval",action:_,target_uri:f,decision:"allow",metadata:{approval_id:m.id}}),E({ok:!0,...m,action:_,target_uri:f,message:`Approved ${_}`},n.json);return}if(a==="audit"){let _=u.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((f)=>({id:f.id,event_type:f.event_type,action:f.action,target_uri:f.target_uri,decision:f.decision,metadata:JSON.parse(f.metadata_json),created_at:f.created_at}));E({ok:!0,events:_,message:`${_.length} audit event(s)`},n.json);return}if(a==="redact"){let _=t.slice(2).join(" ");if(!_)throw Error("Usage: open-knowledge safety redact <text>");let f=de(_,c);if(f.findings.length>0)le(u,{source_uri:"safety://redact",findings:f.findings,metadata:{command:"safety redact"}});R(u,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:f.findings.length>0?"redacted":"allow",metadata:{findings:f.findings.length}}),E({ok:!0,text:f.text,findings:f.findings,message:`Redacted ${f.findings.length} finding(s)`},n.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{u.close()}}if(r==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let o=t[2];if(!o)throw Error("Usage: open-knowledge source resolve <source-ref>");let c=await i.resolveSource(o,{purpose:n.purpose,limit:n.limit});E({ok:!0,...c,message:c.resolved?`Resolved ${c.source_ref} (${c.content.chunks_returned}/${c.content.chunks_total} chunks)`:`Source not indexed: ${o}`},n.json);return}if(r==="ingest"){let a=t[1]??"";if(a==="manifest"){let o=t[2];if(!o)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let c=await i.ingestManifest(o);E({ok:!0,...c,message:`Ingested ${c.items_seen} manifest item(s)`},n.json);return}if(a==="source"){let o=t[2];if(!o)throw Error("Usage: open-knowledge ingest source <source-ref>");let c=await i.ingestSource(o,n.purpose);E({ok:!0,...c,message:`Ingested source ${c.source_ref} (${c.chunks_inserted} chunks)`},n.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(r==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let o=t[2];if(!o)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let c=await i.consumeOutbox(o);E({ok:!0,...c,message:`Consumed ${c.events_seen} outbox event(s)`},n.json);return}if(r==="embeddings"){let a=t[1]??"status";if(a==="status"){let o=i.embeddingStatus();E({ok:!0,...o,message:`${o.total_vector_entries} vector index entries`},n.json);return}if(a==="index"){let o=await i.indexEmbeddings({limit:n.limit,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});E({ok:!0,...o,message:`Embedded ${o.chunks_embedded} chunk(s)`},n.json);return}if(a==="search"){let o=t.slice(2).join(" ");if(!o)throw Error("Usage: open-knowledge embeddings search <query>");let c=await i.semanticSearch({query:o,limit:n.limit,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});E({ok:!0,...c,message:`${c.results.length} semantic result(s)`},n.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(r==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");let o=await i.search({query:a,limit:n.limit,semantic:n.semantic,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});E({ok:!0,...o,message:`${o.results.length} search result(s)`},n.json);return}if(r==="providers"){let a=t[1]??"status";if(a==="status"){let o=i.providerStatus(),c=o.providers.filter((u)=>u.configured).length;E({ok:!0,...o,message:`${c}/${o.providers.length} provider credential(s) configured`},n.json);return}if(a==="models"){let o=i.modelRegistry();E({ok:!0,models:o,message:`${o.length} model alias(es)`},n.json);return}if(a==="check"){let o=t[2]??"default",c=Ne(o,i.config()),u=W(c),_=oe(u.provider,i.config());E({ok:!0,target:o,model_ref:c,provider:u.provider,model:u.model,credential:_,message:`${u.provider} credentials configured`},n.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(xe(s),r==="add"){let a=t[1],o=t[2];if(!a||!o)throw Error("Usage: open-knowledge add <title> <content>");C(s,()=>{let c=L(s),u={id:Se(),title:a,content:o,url:n.url??null,tags:n.tag?[n.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};c.items.push(u),j(s,c),B("info","Item added",{id:u.id,title:u.title}),E({ok:!0,item:u,message:`Added ${u.id}`},n.json)});return}if(r==="list"){if(n.format!==void 0&&n.format!=="table"&&n.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");C(s,()=>{let a=L(s),o=Number.isFinite(n.page)&&n.page>0?n.page:1,c=Number.isFinite(n.limit)&&n.limit>0?n.limit:20,u=n.search?String(n.search).toLowerCase():"",_=n.tag?String(n.tag).toLowerCase():"",f=n.format==="table"||!n.json&&!n.format&&mi(n),m=n.json||n.format==="json",y=a.items;if(n.archived)y=y.filter((h)=>h.archived===!0);else if(!n.includeArchived)y=y.filter((h)=>!h.archived);if(u)y=y.filter((h)=>h.title.toLowerCase().includes(u)||h.content.toLowerCase().includes(u));if(_)y=y.filter((h)=>h.tags&&h.tags.map((G)=>G.toLowerCase()).includes(_));let{sorted:k,sort:g,direction:b}=Ei(y,n),x=(o-1)*c,p=k.slice(x,x+c),A=Math.max(1,Math.ceil(k.length/c));if(m){E({ok:!0,page:o,limit:c,total:k.length,total_pages:A,sort:g,direction:b,items:p},!0);return}if(p.length===0){E(`No items found (search=${u||"none"}, tag=${_||"none"})`,!1);return}if(f){let h=(M)=>M,G=`${h("ID")} ${h("TITLE")} ${h("CREATED")} ${h("URL")} ${h("TAGS")}`;console.log(G);for(let M of p)console.log(`${M.id} ${h(M.title)} ${M.created_at} ${M.url?h(M.url):""} ${M.tags?.length?h(`[${M.tags.join(", ")}]`):""}`);console.log(`Page ${o}/${A} | showing ${p.length} of ${k.length} | sort=${g} ${b} | search=${u||"none"} | tag=${_||"none"}`)}else{for(let h of p)console.log(`${h.id} ${h.title} ${h.created_at}${h.url?` ${h.url}`:""}${h.tags?.length?` [${h.tags.join(", ")}]`:""}`);console.log(`Page ${o}/${A} | showing ${p.length} of ${k.length} | sort=${g} ${b} | search=${u||"none"} | tag=${_||"none"}`)}});return}if(r==="get"){te(n),C(s,()=>{let o=L(s).items.find((c)=>c.id===n.id||c.short_id===n.id);if(!o)throw Error(`Item not found: ${n.id}`);E({ok:!0,item:o,message:`${o.id}: ${o.title}`},n.json)});return}if(r==="update"){te(n),C(s,()=>{let a=L(s),o=a.items.findIndex((u)=>u.id===n.id||u.short_id===n.id);if(o===-1)throw Error(`Item not found: ${n.id}`);let c=a.items[o];if(n.title!==void 0)c.title=n.title;if(n.content!==void 0)c.content=n.content;if(n.url!==void 0)c.url=n.url;if(n.tag!==void 0){if(c.tags=c.tags||[],!c.tags.map((u)=>u.toLowerCase()).includes(n.tag.toLowerCase()))c.tags.push(n.tag)}c.updated_at=new Date().toISOString(),a.items[o]=c,j(s,a),E({ok:!0,item:c,message:`Updated ${c.id}`},n.json)});return}if(r==="archive"||r==="restore"){te(n),C(s,()=>{let a=L(s),o=a.items.findIndex((u)=>u.id===n.id||u.short_id===n.id);if(o===-1)throw Error(`Item not found: ${n.id}`);let c=a.items[o];c.archived=r==="archive",c.updated_at=new Date().toISOString(),a.items[o]=c,j(s,a),E({ok:!0,item:c,message:`${r==="archive"?"Archived":"Restored"} ${c.id}`},n.json)});return}if(r==="untag"){if(te(n),!n.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");C(s,()=>{let a=L(s),o=a.items.findIndex((_)=>_.id===n.id||_.short_id===n.id);if(o===-1)throw Error(`Item not found: ${n.id}`);let c=a.items[o],u=c.tags?.length??0;c.tags=(c.tags??[]).filter((_)=>_.toLowerCase()!==n.tag.toLowerCase()),c.updated_at=new Date().toISOString(),a.items[o]=c,j(s,a),E({ok:!0,item:c,removed:u-c.tags.length,message:`Removed tag from ${c.id}`},n.json)});return}if(r==="upsert"){let a=n.title??t[1],o=n.content??t[2];C(s,()=>{let c=L(s),u=n.id?c.items.findIndex((m)=>m.id===n.id||m.short_id===n.id):-1,_=new Date().toISOString();if(u===-1){if(!a||!o)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let m=n.id??Se(),y={id:m,short_id:Ge(m),title:a,content:o,url:n.url??null,tags:n.tag?[n.tag]:[],metadata:{},archived:!1,created_at:_,updated_at:_};c.items.push(y),j(s,c),E({ok:!0,created:!0,item:y,message:`Upserted ${y.id}`},n.json);return}let f=c.items[u];if(a!==void 0)f.title=a;if(o!==void 0)f.content=o;if(n.url!==void 0)f.url=n.url;if(n.tag!==void 0){if(f.tags=f.tags||[],!f.tags.map((m)=>m.toLowerCase()).includes(n.tag.toLowerCase()))f.tags.push(n.tag)}f.updated_at=_,c.items[u]=f,j(s,c),E({ok:!0,created:!1,item:f,message:`Upserted ${f.id}`},n.json)});return}if(r==="delete"){if(te(n),!n.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");C(s,()=>{let a=L(s),o=a.items.length;a.items=a.items.filter((u)=>u.id!==n.id&&u.short_id!==n.id);let c=o!==a.items.length;if(j(s,a),!c)throw Error(`Item not found: ${n.id}`);B("info","Item deleted",{id:n.id}),E({ok:!0,deleted_id:n.id,message:`Deleted ${n.id}`},n.json)});return}if(r==="export"){let a=n.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");C(s,()=>{let o=L(s);if(a==="jsonl")for(let c of o.items)console.log(JSON.stringify(c));else E({ok:!0,items:o.items},n.json)});return}if(r==="prune"){if(!n.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");C(s,()=>{let a=L(s),o=a.items.length;if(n.olderThan!==void 0){let u=new Date;u.setDate(u.getDate()-n.olderThan),a.items=a.items.filter((_)=>new Date(_.created_at)>=u)}if(n.empty)a.items=a.items.filter((u)=>u.content.trim().length>0);let c=o-a.items.length;j(s,a),B("info","Prune completed",{pruned:c,remaining:a.items.length}),E({ok:!0,pruned:c,remaining:a.items.length,message:`Pruned ${c} item(s)`},n.json)});return}if(r==="dedupe"){if(!n.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");C(s,()=>{let a=L(s),o=new Set,c=a.items.length;a.items=a.items.filter((_)=>{let f=`${_.title}\x00${_.content}`;if(o.has(f))return!1;return o.add(f),!0});let u=c-a.items.length;j(s,a),B("info","Dedupe completed",{removed:u,remaining:a.items.length}),E({ok:!0,removed:u,remaining:a.items.length,message:`Dedupe removed ${u} duplicate(s)`},n.json)});return}if(r==="stats"){C(s,()=>{let a=L(s),o=a.items.filter((b)=>!b.archived),c=o.length,u=a.items.length-c,_=o.filter((b)=>b.url).length,f=o.filter((b)=>b.tags&&b.tags.length>0).length,m=c>0?o.map((b)=>b.created_at).sort()[0]:null,y=c>0?o.map((b)=>b.created_at).sort()[c-1]:null,k={};for(let b of o)for(let x of b.tags||[])k[x]=(k[x]||0)+1;let g=Object.entries(k).sort((b,x)=>x[1]-b[1]).slice(0,5).map(([b,x])=>({tag:b,count:x}));E({ok:!0,total:c,archived:u,with_url:_,with_tags:f,oldest:m,newest:y,top_tags:g,message:`${c} items | ${_} with URL | ${f} with tags`},n.json)});return}let d=gi(t[0]),l=d?` Did you mean '${d}'?`:"";throw B("warn","Unknown command",{input:t[0],suggestion:d}),Error(`Unknown command: ${t[0]}.${l} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)ki(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);B("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{gi as suggestCommand,Ei as sortItems,ki as run,li as parseArgs};
|
|
592
|
+
--empty Remove items with empty content`)}function ts(e){if(e==="add"){console.log("Usage: open-knowledge add <title> <content> [--url <url>] [-t <tag>] [--json]");return}if(e==="list"||e==="ls"){console.log("Usage: open-knowledge list|ls [--format table|json] [-p <page>] [-l <limit>] [-s <search>] [-t <tag>] [--sort created|title] [--desc] [--json]");return}if(e==="get"){console.log("Usage: open-knowledge get --id <id> [--json]");return}if(e==="update"||e==="edit"){console.log("Usage: open-knowledge update|edit --id <id> [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="archive"){console.log("Usage: open-knowledge archive --id <id> [--json]");return}if(e==="restore"||e==="unarchive"){console.log("Usage: open-knowledge restore|unarchive --id <id> [--json]");return}if(e==="upsert"){console.log("Usage: open-knowledge upsert [title] [content] [--id <id>] [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="untag"){console.log("Usage: open-knowledge untag --id <id> -t <tag> [--json]");return}if(e==="delete"||e==="rm"){console.log("Usage: open-knowledge delete|rm --id <id> -y [--json]");return}if(e==="export"){console.log("Usage: open-knowledge export [--format jsonl] [--json]");return}if(e==="prune"){console.log("Usage: open-knowledge prune --yes [--older-than <days>] [--empty] [--json]");return}if(e==="dedupe"){console.log("Usage: open-knowledge dedupe --yes [--json]");return}if(e==="stats"){console.log("Usage: open-knowledge stats [--json]");return}if(e==="paths"){console.log("Usage: open-knowledge paths [--scope local|global|project] [--json]");return}if(e==="storage"){console.log("Usage: open-knowledge storage status|validate [--scope local|global|project] [--json]");return}if(e==="db"){console.log("Usage: open-knowledge db init|stats [--scope local|global|project] [--json]");return}if(e==="wiki"){console.log("Usage: open-knowledge wiki init [--scope local|global|project] [--json]");return}if(e==="source"){console.log("Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]");return}if(e==="ingest"){console.log("Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]");return}if(e==="reindex"){console.log("Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]");return}if(e==="search"){console.log("Usage: open-knowledge search <query> [--context] [--semantic] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="ask"||e==="build"||e==="knowledge"){console.log("Usage: open-knowledge ask|build <prompt> [--generate] [--semantic] [--model default|provider:model] [--approve-write] [--scope local|global|project] [--json]");return}if(e==="embeddings"){console.log("Usage: open-knowledge embeddings status|index|search [query] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="providers"){console.log("Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]");return}if(e==="safety"){console.log("Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]");return}es()}function rs(e){if(e.noColor||process.env.NO_COLOR)return!1;if(process.env.FORCE_COLOR)return!0;return process.stdout.isTTY===!0}function E(e,t,r){if(t){console.log(JSON.stringify(e,null,2));return}if(typeof e==="string"){console.log(e);return}console.log(e.message??JSON.stringify(e,null,2))}function ne(e){if(!e.id)throw Error("Missing required --id. Example: open-knowledge get --id <id>")}function ns(e,t){let r=t.sort??"created";if(r!=="created"&&r!=="title")throw Error("Invalid --sort value. Use 'created' or 'title'.");let n=[...e].sort((i,o)=>{if(r==="title")return i.title.localeCompare(o.title);return i.created_at.localeCompare(o.created_at)});if(t.desc)n.reverse();return{sorted:n,sort:r,direction:t.desc?"desc":"asc"}}async function is(e){let{positional:t,flags:r}=Ji(e);if(H("debug","CLI invoked",{command:t[0],flags:{json:r.json,store:r.store}}),r.version){console.log(r.json?JSON.stringify({name:re.name,version:re.version},null,2):`${re.name} ${re.version}`);return}if(r.completions){let a=r.completions;if(a==="bash")console.log('_open_knowledge() { local cur; cur="${COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --semantic --context --generate --approve-write --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge');else if(a==="zsh")console.log(`#compdef open-knowledge
|
|
593
|
+
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=Yi(t[0]),i=1;if(Zi()&&n&&!cr.includes(n))n="ask",i=0;if(!n||r.help||n==="help"){ts(t[1]);return}let o=or({scope:r.scope}),c=r.store;if(!c)if(r.scope==="project"||r.scope==="local")c=o.jsonStorePath();else c=Ne();if(n==="paths"){E(o.paths(),r.json);return}if(n==="storage"){let a=t[1]??"status";if(a==="status"){let d=o.storageContract(),s=o.validateStorage();E({ok:s.ok,...d,validation:s,message:`${d.storage_type} artifact storage at ${d.artifact_store.uri_prefix}`},r.json);return}if(a==="validate"){let d=o.validateStorage();E({ok:d.ok,validation:d,message:d.ok?"Storage contract valid":`Storage contract invalid: ${d.errors.join("; ")}`},r.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(n==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let s=o.initDb();E({ok:!0,...s,message:`Initialized ${s.path}`},r.json);return}let d=o.dbStats();E({ok:!0,path:o.workspace.knowledgeDbPath,...d,message:`knowledge.db schema v${d.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let d=await o.initWiki();E({ok:!0,...d,message:`Initialized wiki layout in ${o.workspace.home}`},r.json);return}if(n==="safety"){let a=t[1]??"status",d=o.ensureWorkspace(),s=o.safetyPolicy();o.initDb();let l=w(d.knowledgeDbPath);try{if(a==="status"){E({ok:!0,mode:s.mode,workspace:d.home,allow_write_roots:s.allowWriteRoots,read_only_source_access:s.readOnlySourceAccess,network:s.network,redaction:s.redaction,approvals:s.approvals,message:`Safety policy: ${s.mode}`},r.json);return}if(a==="check"){let f=t[2]??"generated_write",g=t[3]??null,k;try{if(f==="web_search")me(s),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!g)throw Error("safety check s3_read requires an s3:// target.");X(g,s),k={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"}}else k=qt(l,s,f,g);A(l,{event_type:"safety_check",action:f,target_uri:g,decision:k.decision==="allow"?"allow":"requires_approval",metadata:k}),E({ok:!0,...k,message:`Safety check ${k.decision}`},r.json);return}catch(h){throw A(l,{event_type:"safety_check",action:f,target_uri:g,decision:"deny",metadata:{error:h instanceof Error?h.message:String(h)}}),h}}if(a==="approve"){let f=t[2]??"generated_write",g=t[3]??null,k=$t(l,{action:f,target_uri:g,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});A(l,{event_type:"approval",action:f,target_uri:g,decision:"allow",metadata:{approval_id:k.id}}),E({ok:!0,...k,action:f,target_uri:g,message:`Approved ${f}`},r.json);return}if(a==="audit"){let f=l.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((g)=>({id:g.id,event_type:g.event_type,action:g.action,target_uri:g.target_uri,decision:g.decision,metadata:JSON.parse(g.metadata_json),created_at:g.created_at}));E({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(a==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let g=ke(f,s);if(g.findings.length>0)Ee(l,{source_uri:"safety://redact",findings:g.findings,metadata:{command:"safety redact"}});A(l,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:g.findings.length>0?"redacted":"allow",metadata:{findings:g.findings.length}}),E({ok:!0,text:g.text,findings:g.findings,message:`Redacted ${g.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{l.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge source resolve <source-ref>");let s=await o.resolveSource(d,{purpose:r.purpose,limit:r.limit});E({ok:!0,...s,message:s.resolved?`Resolved ${s.source_ref} (${s.content.chunks_returned}/${s.content.chunks_total} chunks)`:`Source not indexed: ${d}`},r.json);return}if(n==="ingest"){let a=t[1]??"";if(a==="manifest"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let s=await o.ingestManifest(d);E({ok:!0,...s,message:`Ingested ${s.items_seen} manifest item(s)`},r.json);return}if(a==="source"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest source <source-ref>");let s=await o.ingestSource(d,r.purpose);E({ok:!0,...s,message:`Ingested source ${s.source_ref} (${s.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let s=await o.consumeOutbox(d);E({ok:!0,...s,message:`Consumed ${s.events_seen} outbox event(s)`},r.json);return}if(n==="embeddings"){let a=t[1]??"status";if(a==="status"){let d=o.embeddingStatus();E({ok:!0,...d,message:`${d.total_vector_entries} vector index entries`},r.json);return}if(a==="index"){let d=await o.indexEmbeddings({limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`Embedded ${d.chunks_embedded} chunk(s)`},r.json);return}if(a==="search"){let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge embeddings search <query>");let s=await o.semanticSearch({query:d,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...s,message:`${s.results.length} semantic result(s)`},r.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(n==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");if(r.context){let s=await o.retrieveContext({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...s,message:`${s.excerpts.length} context excerpt(s)`},r.json);return}let d=await o.search({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});E({ok:!0,...d,message:`${d.results.length} search result(s)`},r.json);return}if(n==="ask"||n==="build"){let a=t.slice(i).join(" ");if(!a)throw Error("Usage: open-knowledge ask <prompt>");let d=await o.runPrompt({prompt:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake,generate:r.generate,approveWrite:r.approveWrite});E({ok:!0,...d,message:d.generated?"Generated answer with citations":"Prepared citation context draft"},r.json);return}if(n==="providers"){let a=t[1]??"status";if(a==="status"){let d=o.providerStatus(),s=d.providers.filter((l)=>l.configured).length;E({ok:!0,...d,message:`${s}/${d.providers.length} provider credential(s) configured`},r.json);return}if(a==="models"){let d=o.modelRegistry();E({ok:!0,models:d,message:`${d.length} model alias(es)`},r.json);return}if(a==="check"){let d=t[2]??"default",s=B(d,o.config()),l=M(s),f=ee(l.provider,o.config());E({ok:!0,target:d,model_ref:s,provider:l.provider,model:l.model,credential:f,message:`${l.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(Ae(c),n==="add"){let a=t[1],d=t[2];if(!a||!d)throw Error("Usage: open-knowledge add <title> <content>");D(c,()=>{let s=C(c),l={id:Ie(),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};s.items.push(l),K(c,s),H("info","Item added",{id:l.id,title:l.title}),E({ok:!0,item:l,message:`Added ${l.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");D(c,()=>{let a=C(c),d=Number.isFinite(r.page)&&r.page>0?r.page:1,s=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,l=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",g=r.format==="table"||!r.json&&!r.format&&rs(r),k=r.json||r.format==="json",h=a.items;if(r.archived)h=h.filter((b)=>b.archived===!0);else if(!r.includeArchived)h=h.filter((b)=>!b.archived);if(l)h=h.filter((b)=>b.title.toLowerCase().includes(l)||b.content.toLowerCase().includes(l));if(f)h=h.filter((b)=>b.tags&&b.tags.map((xe)=>xe.toLowerCase()).includes(f));let{sorted:m,sort:x,direction:y}=ns(h,r),p=(d-1)*s,S=m.slice(p,p+s),R=Math.max(1,Math.ceil(m.length/s));if(k){E({ok:!0,page:d,limit:s,total:m.length,total_pages:R,sort:x,direction:y,items:S},!0);return}if(S.length===0){E(`No items found (search=${l||"none"}, tag=${f||"none"})`,!1);return}if(g){let b=(F)=>F,xe=`${b("ID")} ${b("TITLE")} ${b("CREATED")} ${b("URL")} ${b("TAGS")}`;console.log(xe);for(let F of S)console.log(`${F.id} ${b(F.title)} ${F.created_at} ${F.url?b(F.url):""} ${F.tags?.length?b(`[${F.tags.join(", ")}]`):""}`);console.log(`Page ${d}/${R} | showing ${S.length} of ${m.length} | sort=${x} ${y} | search=${l||"none"} | tag=${f||"none"}`)}else{for(let b of S)console.log(`${b.id} ${b.title} ${b.created_at}${b.url?` ${b.url}`:""}${b.tags?.length?` [${b.tags.join(", ")}]`:""}`);console.log(`Page ${d}/${R} | showing ${S.length} of ${m.length} | sort=${x} ${y} | search=${l||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){ne(r),D(c,()=>{let d=C(c).items.find((s)=>s.id===r.id||s.short_id===r.id);if(!d)throw Error(`Item not found: ${r.id}`);E({ok:!0,item:d,message:`${d.id}: ${d.title}`},r.json)});return}if(n==="update"){ne(r),D(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let s=a.items[d];if(r.title!==void 0)s.title=r.title;if(r.content!==void 0)s.content=r.content;if(r.url!==void 0)s.url=r.url;if(r.tag!==void 0){if(s.tags=s.tags||[],!s.tags.map((l)=>l.toLowerCase()).includes(r.tag.toLowerCase()))s.tags.push(r.tag)}s.updated_at=new Date().toISOString(),a.items[d]=s,K(c,a),E({ok:!0,item:s,message:`Updated ${s.id}`},r.json)});return}if(n==="archive"||n==="restore"){ne(r),D(c,()=>{let a=C(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let s=a.items[d];s.archived=n==="archive",s.updated_at=new Date().toISOString(),a.items[d]=s,K(c,a),E({ok:!0,item:s,message:`${n==="archive"?"Archived":"Restored"} ${s.id}`},r.json)});return}if(n==="untag"){if(ne(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");D(c,()=>{let a=C(c),d=a.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let s=a.items[d],l=s.tags?.length??0;s.tags=(s.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),s.updated_at=new Date().toISOString(),a.items[d]=s,K(c,a),E({ok:!0,item:s,removed:l-s.tags.length,message:`Removed tag from ${s.id}`},r.json)});return}if(n==="upsert"){let a=r.title??t[1],d=r.content??t[2];D(c,()=>{let s=C(c),l=r.id?s.items.findIndex((k)=>k.id===r.id||k.short_id===r.id):-1,f=new Date().toISOString();if(l===-1){if(!a||!d)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let k=r.id??Ie(),h={id:k,short_id:nt(k),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};s.items.push(h),K(c,s),E({ok:!0,created:!0,item:h,message:`Upserted ${h.id}`},r.json);return}let g=s.items[l];if(a!==void 0)g.title=a;if(d!==void 0)g.content=d;if(r.url!==void 0)g.url=r.url;if(r.tag!==void 0){if(g.tags=g.tags||[],!g.tags.map((k)=>k.toLowerCase()).includes(r.tag.toLowerCase()))g.tags.push(r.tag)}g.updated_at=f,s.items[l]=g,K(c,s),E({ok:!0,created:!1,item:g,message:`Upserted ${g.id}`},r.json)});return}if(n==="delete"){if(ne(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");D(c,()=>{let a=C(c),d=a.items.length;a.items=a.items.filter((l)=>l.id!==r.id&&l.short_id!==r.id);let s=d!==a.items.length;if(K(c,a),!s)throw Error(`Item not found: ${r.id}`);H("info","Item deleted",{id:r.id}),E({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let a=r.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");D(c,()=>{let d=C(c);if(a==="jsonl")for(let s of d.items)console.log(JSON.stringify(s));else E({ok:!0,items:d.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");D(c,()=>{let a=C(c),d=a.items.length;if(r.olderThan!==void 0){let l=new Date;l.setDate(l.getDate()-r.olderThan),a.items=a.items.filter((f)=>new Date(f.created_at)>=l)}if(r.empty)a.items=a.items.filter((l)=>l.content.trim().length>0);let s=d-a.items.length;K(c,a),H("info","Prune completed",{pruned:s,remaining:a.items.length}),E({ok:!0,pruned:s,remaining:a.items.length,message:`Pruned ${s} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");D(c,()=>{let a=C(c),d=new Set,s=a.items.length;a.items=a.items.filter((f)=>{let g=`${f.title}\x00${f.content}`;if(d.has(g))return!1;return d.add(g),!0});let l=s-a.items.length;K(c,a),H("info","Dedupe completed",{removed:l,remaining:a.items.length}),E({ok:!0,removed:l,remaining:a.items.length,message:`Dedupe removed ${l} duplicate(s)`},r.json)});return}if(n==="stats"){D(c,()=>{let a=C(c),d=a.items.filter((y)=>!y.archived),s=d.length,l=a.items.length-s,f=d.filter((y)=>y.url).length,g=d.filter((y)=>y.tags&&y.tags.length>0).length,k=s>0?d.map((y)=>y.created_at).sort()[0]:null,h=s>0?d.map((y)=>y.created_at).sort()[s-1]:null,m={};for(let y of d)for(let p of y.tags||[])m[p]=(m[p]||0)+1;let x=Object.entries(m).sort((y,p)=>p[1]-y[1]).slice(0,5).map(([y,p])=>({tag:y,count:p}));E({ok:!0,total:s,archived:l,with_url:f,with_tags:g,oldest:k,newest:h,top_tags:x,message:`${s} items | ${f} with URL | ${g} with tags`},r.json)});return}let u=Qi(t[0]),_=u?` Did you mean '${u}'?`:"";throw H("warn","Unknown command",{input:t[0],suggestion:u}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)is(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);H("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{Qi as suggestCommand,ns as sortItems,is as run,Ji as parseArgs};
|