@hasna/knowledge 0.2.10 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/bin/open-knowledge-mcp.js +169 -2
- package/bin/open-knowledge.js +32 -31
- package/docs/architecture/ai-native-knowledge-base.md +15 -0
- package/package.json +6 -2
- package/src/cli.ts +31 -4
- package/src/mcp.js +14 -0
- package/src/providers.ts +308 -0
- package/src/service.ts +9 -0
- package/src/workspace.ts +41 -0
package/bin/open-knowledge.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
|
-
var A=import.meta.require;import{readFileSync as V,writeFileSync as J,existsSync as G,renameSync as
|
|
4
|
-
`);return t}function
|
|
3
|
+
var A=import.meta.require;import{readFileSync as V,writeFileSync as J,existsSync as G,renameSync as _t,unlinkSync as xe}from"fs";import{randomUUID as ve}from"crypto";import{existsSync as st,mkdirSync as ce,readFileSync as ot,writeFileSync as at}from"fs";import{homedir as be}from"os";import{dirname as ct,join as S,resolve as ut}from"path";var dt=S(".hasna","apps","knowledge");function ue(){return S(be(),".open-knowledge","db.json")}function de(){return S(be(),".hasna","apps","knowledge")}function lt(e=process.cwd()){return ut(e,dt)}function B(e){return{home:e,configPath:S(e,"config.json"),jsonStorePath:S(e,"db.json"),knowledgeDbPath:S(e,"knowledge.db"),artifactsDir:S(e,"artifacts"),cacheDir:S(e,"cache"),exportsDir:S(e,"exports"),indexesDir:S(e,"indexes"),logsDir:S(e,"logs"),runsDir:S(e,"runs"),schemasDir:S(e,"schemas"),wikiDir:S(e,"wiki")}}function ft(){return{version:1,mode:"local",storage:{type:"local",artifacts_root:"artifacts"},sources:{preferred_ref:"open-files",allowed_schemes:["open-files","s3","file","https","http"]},providers:{default_model:"openai:gpt-5.2",aliases:{fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"},openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},safety:{network:{web_search_enabled:!1,s3_reads_enabled:!1,allowed_s3_buckets:[]},redaction:{enabled:!0},approvals:{generated_writes_require_approval:!0}}}}function we(e){let t=B(e);ce(t.home,{recursive:!0});for(let r of[t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir])ce(r,{recursive:!0});if(!st(t.configPath))at(t.configPath,`${JSON.stringify(ft(),null,2)}
|
|
4
|
+
`);return t}function Re(e,t=process.cwd()){if(e==="project"||e==="local")return B(lt(t));return B(de())}function Y(e){ce(ct(e),{recursive:!0})}function Se(e){let t=ot(e,"utf8");return JSON.parse(t)}function le(){return B(de()).jsonStorePath}function fe(e){if(!G(e))if(Y(e),e===le()&&G(ue()))J(e,V(ue(),"utf8"));else J(e,JSON.stringify({items:[]},null,2))}function pt(e){return`${e}.lock`}function gt(e,t){let i=Date.now();while(Date.now()-i<5000){try{if(!G(e)){J(e,JSON.stringify({owner:t,ts:Date.now()}));return}let l=JSON.parse(V(e,"utf8"));if(Date.now()-l.ts>1e4)xe(e)}catch{}let s=Date.now();while(Date.now()-s<50);}throw Error(`Could not acquire lock on ${e} after 5000ms`)}function Et(e,t){try{if(G(e)){if(JSON.parse(V(e,"utf8")).owner===t)xe(e)}}catch{}}function x(e){fe(e);let t=V(e,"utf8"),r=JSON.parse(t);if(!r||!Array.isArray(r.items))return{items:[]};return r}function D(e,t){let r=`${e}.tmp.${ve()}`;J(r,JSON.stringify(t,null,2)),_t(r,e)}function v(e,t){let r=ve(),n=pt(e);gt(n,r);try{return t()}finally{Et(n,r)}}function _e(){return`k_${Date.now().toString(36)}_${Math.random().toString(36).slice(2,8)}`}function Oe(e){return e.replace(/^k_/,"").slice(0,12)}import{Database as ht}from"bun:sqlite";var yt=`
|
|
5
5
|
PRAGMA journal_mode = WAL;
|
|
6
6
|
PRAGMA foreign_keys = ON;
|
|
7
7
|
|
|
@@ -168,7 +168,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
168
168
|
|
|
169
169
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
170
170
|
VALUES (1, datetime('now'));
|
|
171
|
-
`,
|
|
171
|
+
`,Tt=`
|
|
172
172
|
DROP TABLE IF EXISTS chunks_fts;
|
|
173
173
|
|
|
174
174
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
@@ -181,7 +181,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
181
181
|
|
|
182
182
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
183
183
|
VALUES (2, datetime('now'));
|
|
184
|
-
`,
|
|
184
|
+
`,mt=`
|
|
185
185
|
CREATE TABLE IF NOT EXISTS audit_events (
|
|
186
186
|
id TEXT PRIMARY KEY,
|
|
187
187
|
event_type TEXT NOT NULL,
|
|
@@ -212,59 +212,59 @@ CREATE INDEX IF NOT EXISTS idx_approval_gates_status ON approval_gates(status);
|
|
|
212
212
|
|
|
213
213
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
214
214
|
VALUES (3, datetime('now'));
|
|
215
|
-
`;function C(e){Y(e);let t=new
|
|
216
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[
|
|
217
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${Ee()}`,t.source_uri??null,t.run_id??null,
|
|
218
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
215
|
+
`;function C(e){Y(e);let t=new ht(e);return t.exec("PRAGMA foreign_keys = ON;"),t}function U(e){let t=C(e);try{if(t.exec(yt),Q(t)<2)t.exec(Tt);if(Q(t)<3)t.exec(mt);return{path:e,schema_version:Q(t)}}finally{t.close()}}function Q(e){return e.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0}function I(e,t){return e.query(`SELECT COUNT(*) AS n FROM ${t}`).get()?.n??0}function Ne(e){let t=C(e);try{return{schema_version:Q(t),sources:I(t,"sources"),source_revisions:I(t,"source_revisions"),chunks:I(t,"chunks"),wiki_pages:I(t,"wiki_pages"),citations:I(t,"citations"),indexes:I(t,"knowledge_indexes"),runs:I(t,"runs"),run_events:I(t,"run_events"),redaction_findings:I(t,"redaction_findings"),audit_events:I(t,"audit_events"),approval_gates:I(t,"approval_gates")}}finally{t.close()}}import{existsSync as kt,mkdirSync as Le,readFileSync as bt,writeFileSync as wt}from"fs";import{dirname as Rt,join as pe,relative as St,sep as xt}from"path";function z(e){let t=e.replace(/\\/g,"/").trim();if(!t||t.startsWith("/"))throw Error(`Invalid artifact key: ${e}`);let r=t.split("/").filter(Boolean);if(r.length===0||r.some((n)=>n==="."||n===".."))throw Error(`Invalid artifact key: ${e}`);return r.join("/")}function ge(e,t){let r=St(e,t);if(r.startsWith("..")||r===".."||r.startsWith(`..${xt}`))throw Error(`Artifact path escapes root: ${t}`)}class Ae{root;type="local";canRead=!0;canWrite=!0;constructor(e){this.root=e;Le(e,{recursive:!0})}async put(e){let t=z(e.key),r=pe(this.root,t);return ge(this.root,r),Le(Rt(r),{recursive:!0}),wt(r,e.body),{key:t,uri:`file://${r}`}}async getText(e){let t=z(e),r=pe(this.root,t);return ge(this.root,r),bt(r,"utf8")}async exists(e){let t=z(e),r=pe(this.root,t);return ge(this.root,r),kt(r)}}class Ie{options;type="s3";canRead=!0;canWrite=!0;client;constructor(e){this.options=e;this.client=e.client}async getClient(){if(this.client)return this.client;let[{S3Client:e},{fromIni:t}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]);return this.client=new e({region:this.options.region,credentials:this.options.profile?t({profile:this.options.profile}):void 0,maxAttempts:this.options.max_attempts}),this.client}objectKey(e){let t=z(e),r=this.options.prefix?z(this.options.prefix):"";return r?`${r}/${t}`:t}async put(e){let[{PutObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e.key);return await r.send(new t({Bucket:this.options.bucket,Key:n,Body:e.body,ContentType:e.content_type,Metadata:e.metadata,ServerSideEncryption:this.options.server_side_encryption,SSEKMSKeyId:this.options.kms_key_id})),{key:n,uri:`s3://${this.options.bucket}/${n}`}}async getText(e){let[{GetObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e),i=await r.send(new t({Bucket:this.options.bucket,Key:n}));if(!i.Body)return"";return await i.Body.transformToString()}async exists(e){let[{HeadObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e);try{return await r.send(new t({Bucket:this.options.bucket,Key:n})),!0}catch(i){let s=i instanceof Error?i.name:"";if(s==="NotFound"||s==="NoSuchKey"||s==="NotFoundError")return!1;throw i}}}function De(e,t){if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)throw Error("S3 artifact storage requires storage.s3.bucket");return new Ie({bucket:e.storage.s3.bucket,prefix:e.storage.s3.prefix,region:e.storage.s3.region,profile:e.storage.s3.profile,max_attempts:e.storage.s3.max_attempts,server_side_encryption:e.storage.s3.server_side_encryption,kms_key_id:e.storage.s3.kms_key_id})}return new Ae(t.artifactsDir)}import{createHash as jt,randomUUID as Ft}from"crypto";import{existsSync as Xt,readFileSync as Mt}from"fs";import{basename as Wt}from"path";function Ce(e,t){if(!e)throw Error(t);return e}function vt(e){let r=e.slice(13).split("/").filter(Boolean),n=r[0];if(n!=="file"&&n!=="source")throw Error("Invalid open-files ref. Expected open-files://file/<id>, open-files://file/<id>/revision/<revision_id>, or open-files://source/<id>/path/<path>.");let i=Ce(r[1],"Invalid open-files ref. Missing id.");if(n==="file"){if(r.length===2)return{kind:"open-files",uri:e,entity:n,id:i};if(r[2]==="revision"&&r[3]&&r.length===4)return{kind:"open-files",uri:e,entity:n,id:i,revision_id:decodeURIComponent(r[3])};throw Error("Invalid open-files file ref. Expected open-files://file/<id>/revision/<revision_id>.")}let s=r.indexOf("path"),l=s>=0?decodeURIComponent(r.slice(s+1).join("/")):void 0;return{kind:"open-files",uri:e,entity:n,id:i,path:l}}function Ot(e){let t=new URL(e),r=Ce(t.hostname,"Invalid s3 ref. Missing bucket."),n=decodeURIComponent(t.pathname.replace(/^\/+/,""));if(!n)throw Error("Invalid s3 ref. Missing object key.");return{kind:"s3",uri:e,bucket:r,key:n}}function Nt(e){let t=new URL(e);return{kind:"file",uri:e,path:decodeURIComponent(t.pathname)}}function Lt(e){let t=new URL(e);return{kind:"web",uri:e,url:t.toString()}}function N(e){if(e.startsWith("open-files://"))return vt(e);if(e.startsWith("s3://"))return Ot(e);if(e.startsWith("file://"))return Nt(e);if(e.startsWith("https://")||e.startsWith("http://"))return Lt(e);throw Error(`Unsupported source ref scheme: ${e}`)}function Ue(e,t=N(e)){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Pe(e){let t=N(e);return t.kind==="open-files"&&t.entity==="file"?t.revision_id??null:null}import{createHash as At,randomUUID as Ee}from"crypto";import{relative as It,resolve as je,sep as Dt}from"path";function Ke(e){let t=process.env[e];return t==="1"||t==="true"||t==="yes"}function Fe(e,t){let r=e,n=new Set(r.safety?.network?.allowed_s3_buckets??[]);if(e.storage.type==="s3"&&e.storage.s3?.bucket)n.add(e.storage.s3.bucket);if(process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS)for(let i of process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.split(",").map((s)=>s.trim()).filter(Boolean))n.add(i);return{mode:e.mode,allowWriteRoots:[t.home,t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir].map((i)=>je(i)),readOnlySourceAccess:!0,network:{webSearchEnabled:r.safety?.network?.web_search_enabled??Ke("HASNA_KNOWLEDGE_WEB_SEARCH"),s3ReadsEnabled:r.safety?.network?.s3_reads_enabled??Ke("HASNA_KNOWLEDGE_ALLOW_S3_READS"),allowedS3Buckets:[...n].sort()},redaction:{enabled:r.safety?.redaction?.enabled??!0},approvals:{generatedWritesRequireApproval:r.safety?.approvals?.generated_writes_require_approval??!0}}}function Ct(e,t){let r=It(e,t);return r===""||!r.startsWith("..")&&r!==".."&&!r.startsWith(`..${Dt}`)}function j(e,t){let r=je(e);if(!t.allowWriteRoots.some((n)=>Ct(n,r)))throw Error(`Safety policy denied write outside .hasna/apps/knowledge: ${e}`)}function K(e,t){let n=new URL(e).hostname;if(!t.network.s3ReadsEnabled)throw Error("Safety policy denied S3 read. Set safety.network.s3_reads_enabled=true or HASNA_KNOWLEDGE_ALLOW_S3_READS=1.");if(!t.network.allowedS3Buckets.includes(n))throw Error(`Safety policy denied S3 bucket "${n}". Add it to safety.network.allowed_s3_buckets or HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.`)}function Z(e){if(!e.network.webSearchEnabled)throw Error("Safety policy denied web search. Set safety.network.web_search_enabled=true or HASNA_KNOWLEDGE_WEB_SEARCH=1.")}var Ut=[{type:"private_key_block",severity:"high",regex:/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,replacement:"[REDACTED:private_key_block]"},{type:"secret_assignment",severity:"high",regex:/\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*['"]?[^'"\s]{8,}/gi,replacement:"[REDACTED:secret_assignment]"},{type:"openai_api_key",severity:"high",regex:/\bsk-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:openai_api_key]"},{type:"anthropic_api_key",severity:"high",regex:/\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:anthropic_api_key]"},{type:"aws_access_key_id",severity:"high",regex:/\bA(?:KIA|SIA)[A-Z0-9]{16}\b/g,replacement:"[REDACTED:aws_access_key_id]"}];function ee(e,t){if(t&&!t.redaction.enabled)return{text:e,findings:[]};let r=e,n=[];for(let i of Ut)r=r.replace(i.regex,(s,...l)=>{let _=typeof l.at(-2)==="number"?l.at(-2):r.indexOf(s);return n.push({type:i.type,severity:i.severity,start:Math.max(0,_),end:Math.max(0,_+s.length)}),i.replacement});return{text:r,findings:n}}function Pt(e){return`audit_${At("sha256").update(`${e.event_type}\x00${e.action}\x00${e.target_uri??""}\x00${e.created_at??""}\x00${JSON.stringify(e.metadata??{})}\x00${Ee()}`).digest("hex").slice(0,24)}`}function w(e,t){let r=t.created_at??new Date().toISOString(),n=Pt({...t,created_at:r});return e.run(`INSERT INTO audit_events (id, event_type, action, target_uri, decision, metadata_json, created_at)
|
|
216
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function te(e,t){let r=t.created_at??new Date().toISOString();for(let n of t.findings)e.run(`INSERT INTO redaction_findings (id, source_uri, run_id, severity, finding_type, metadata_json, created_at)
|
|
217
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${Ee()}`,t.source_uri??null,t.run_id??null,n.severity,n.type,JSON.stringify({...t.metadata??{},start:n.start,end:n.end}),r]);return t.findings.length}function Xe(e,t){let r=t.created_at??new Date().toISOString(),n=`approval_${Ee()}`;return e.run(`INSERT INTO approval_gates (id, action, target_uri, status, reason, approved_by, metadata_json, created_at, updated_at)
|
|
218
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function Kt(e,t,r){let n=e.query(`SELECT id FROM approval_gates
|
|
219
219
|
WHERE action = ? AND status = 'approved' AND (target_uri IS NULL OR target_uri = ? OR ? IS NULL)
|
|
220
|
-
ORDER BY updated_at DESC LIMIT 1`).get(t,
|
|
220
|
+
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function Me(e,t,r,n){let i=r==="generated_write"&&t.approvals.generatedWritesRequireApproval,s=!i||Kt(e,r,n);return{action:r,target_uri:n??null,approval_required:i,approved:s,decision:s?"allow":"requires_approval"}}function re(e,t){return`${e}_${jt("sha256").update(t).digest("hex").slice(0,20)}`}function X(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function b(e){return typeof e==="string"&&e.length>0?e:void 0}function $t(e){let t=b(e.source_ref)??b(e.source_uri)??b(e.uri);if(t)return t;let r=b(e.file_id);if(r){let s=b(e.revision_id)??b(e.revision),l=`open-files://file/${encodeURIComponent(r)}`;return s?`${l}/revision/${encodeURIComponent(s)}`:l}let n=b(e.source_id),i=b(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Outbox event is missing source_ref, file_id, or source_id/path.")}function Bt(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function zt(e){return b(e.hash)??b(e.checksum)??b(e.sha256)??null}function Ht(e,t,r){return b(e.revision_id)??b(e.revision)??b(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??null}function qt(e){return(b(e.event)??b(e.type)??b(e.action)??b(e.change_type)??"changed").toLowerCase()}function Yt(e){let t=b(e.path);return b(e.title)??b(e.name)??(t?Wt(t):null)}function Jt(e,t){let r=$t(e),n=N(r),i=zt(e);return{raw:e,eventType:qt(e),sourceRef:r,sourceUri:Bt(r,n),kind:n.kind,title:Yt(e),revision:Ht(e,n,i),hash:i,status:b(e.status)?.toLowerCase()??null,updatedAt:b(e.updated_at)??t,acl:e.permissions??e.acl??void 0}}function Gt(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Outbox array parse failed.");return r.map((n)=>{let i=X(n);if(!i)throw Error("Outbox array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=X(r);if(!n)throw Error("Outbox object parse failed.");if(Array.isArray(n.events))return n.events.map((i)=>{let s=X(i);if(!s)throw Error("Outbox events entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=X(JSON.parse(i));if(!s)throw Error("Outbox JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=X(JSON.parse(r));if(!n)throw Error("Outbox JSONL entries must be objects.");return n})}async function Vt(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 outbox URI: ${e}`);if(r)K(e,r);let[{S3Client:l,GetObjectCommand:_},{fromIni:o}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new l({region:a?.region,credentials:a?.profile?o({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new _({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function Qt(e,t,r){if(e.startsWith("s3://"))return Vt(e,t,r);if(!Xt(e))throw Error(`Outbox not found: ${e}`);return Mt(e,"utf8")}function We(e,t){let r={};if(e)try{r=X(JSON.parse(e))??{}}catch{r={}}return JSON.stringify({...r,...t})}function Zt(e,t,r){let n=re("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
221
221
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
222
222
|
ON CONFLICT(uri) DO UPDATE SET
|
|
223
223
|
kind = excluded.kind,
|
|
224
224
|
title = COALESCE(excluded.title, sources.title),
|
|
225
|
-
updated_at = excluded.updated_at`,[
|
|
225
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(b(t.raw.path))s.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[We(i.metadata_json,s),t.acl===void 0?null:JSON.stringify(t.acl),t.acl===void 0?null:JSON.stringify(t.acl),t.updatedAt,i.id]),i.id}function er(e,t,r,n){if(!r.revision)return null;let i=re("rev",`${t}\x00${r.revision}`),s={source_ref:r.sourceRef,source_uri:r.sourceUri,status:r.status,last_outbox_event:r.eventType,reindex_required:!0};return e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
226
226
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
227
227
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
228
228
|
hash = COALESCE(excluded.hash, source_revisions.hash),
|
|
229
|
-
metadata_json = excluded.metadata_json`,[i,t,
|
|
230
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:
|
|
231
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[
|
|
232
|
-
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[
|
|
233
|
-
`);if(!
|
|
229
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,b(r.raw.extracted_text_ref)??null,JSON.stringify(s),n]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision)?.id??null}function tr(e,t,r){if(r.revision)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").all(t,r.revision).map((n)=>n.id);if(r.hash)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND hash = ?").all(t,r.hash).map((n)=>n.id);return e.query("SELECT id FROM source_revisions WHERE source_id = ?").all(t).map((n)=>n.id)}function rr(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t),n=0;for(let s of r){let l=e.query("SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?").get(s.id);n+=l?.n??0,e.run("DELETE FROM chunk_embeddings WHERE chunk_id = ?",[s.id]),e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[s.id])}e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]);let i=e.query("SELECT metadata_json FROM source_revisions WHERE id = ?").get(t);return e.run("UPDATE source_revisions SET metadata_json = ? WHERE id = ?",[We(i?.metadata_json,{reindex_required:!0,invalidated_at:new Date().toISOString()}),t]),{chunksDeleted:r.length,embeddingsDeleted:n}}function nr(e,t){return t==="deleted"||["delete","deleted","remove","removed"].includes(e)}function ir(e){return["move","moved","rename","renamed","path_changed"].includes(e)}function sr(e){return["permission","permissions","permission_changed","acl_changed"].includes(e)}async function $e(e){let t=(e.now??new Date).toISOString();if(e.safetyPolicy)j(e.dbPath,e.safetyPolicy);U(e.dbPath);let r=await Qt(e.input,e.config,e.safetyPolicy),n=Gt(r),i=C(e.dbPath),s=`run_${Ft()}`;try{return i.transaction(()=>{i.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
230
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:n.length}),t,t]);let l=new Set,_=new Set,o=0,a=0,c=0,u=0,f=0,d=0;return w(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:n.length,read_only:!0},created_at:t}),n.forEach((y,T)=>{let p=Jt(y,t),L=Zt(i,p,t);l.add(L);let k=er(i,L,p,t);if(k)_.add(k);let R=tr(i,L,p);for(let g of R){_.add(g);let O=rr(i,g);o+=O.chunksDeleted,a+=O.embeddingsDeleted,c+=1}if(nr(p.eventType,p.status))u+=1;if(ir(p.eventType))f+=1;if(sr(p.eventType)||p.acl!==void 0)d+=1;i.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
231
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[re("evt",`${s}\x00${T}\x00${p.sourceRef}\x00${p.eventType}`),s,"info",p.eventType,JSON.stringify({source_ref:p.sourceRef,source_uri:p.sourceUri,revision:p.revision,hash:p.hash,status:p.status,affected_revisions:R.length}),p.updatedAt])}),i.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
232
|
+
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[re("usage",s),s,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),w(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:s,events:n.length,sources:l.size,revisions:_.size,chunks_deleted:o,embeddings_deleted:a},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:s,events_seen:n.length,sources_touched:l.size,revisions_touched:_.size,chunks_deleted:o,embeddings_deleted:a,stale_revisions:c,deleted_sources:u,moved_sources:f,permission_updates:d}})()}finally{i.close()}}import{createHash as or}from"crypto";import{existsSync as ar,readFileSync as cr}from"fs";import{basename as ur}from"path";function he(e,t){return`${e}_${or("sha256").update(t).digest("hex").slice(0,20)}`}function M(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function E(e){return typeof e==="string"&&e.length>0?e:void 0}function dr(e){return typeof e==="number"&&Number.isFinite(e)?e:void 0}function lr(e){let t=E(e.source_ref)??E(e.source_uri)??E(e.uri);if(t)return t;let r=E(e.file_id);if(r){let s=E(e.revision_id)??E(e.revision),l=`open-files://file/${encodeURIComponent(r)}`;return s?`${l}/revision/${encodeURIComponent(s)}`:l}let n=E(e.source_id),i=E(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Manifest item is missing source_ref, file_id, or source_id/path.")}function fr(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function _r(e){let t=E(e.extracted_text)??E(e.text)??E(e.content_text)??E(e.markdown);if(t!==void 0)return t;let r=e.content;return typeof r==="string"?r:null}function pr(e){let t=E(e.extracted_text_ref)??E(e.extracted_text_uri)??E(e.text_ref);if(t)return t;let r=M(e.content);return E(r?.extracted_text_ref)??E(r?.extracted_text_uri)??null}function gr(e){let t=E(e.path);return E(e.title)??E(e.name)??(t?ur(t):null)}function Er(e){return E(e.hash)??E(e.checksum)??E(e.sha256)??null}function hr(e,t,r){return E(e.revision_id)??E(e.revision)??E(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??E(e.updated_at)??"current"}function yr(e,t){let r={};for(let[n,i]of Object.entries(e)){if(["text","content","content_text","extracted_text","markdown"].includes(n))continue;r[n]=i}return r.source_ref=t.sourceRef,r.source_uri=t.sourceUri,r.status=t.status,r}function Tr(e,t){let r=lr(e),n=N(r),i=fr(r,n),s=Er(e),l=E(e.status)??"active";return{raw:e,sourceRef:r,sourceUri:i,kind:n.kind,title:gr(e),revision:hr(e,n,s),hash:s,extractedTextUri:pr(e),text:_r(e),metadata:yr(e,{sourceRef:r,sourceUri:i,status:l}),acl:e.permissions??e.acl??{},status:l,updatedAt:E(e.updated_at)??t}}function mr(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Manifest array parse failed.");return r.map((n)=>{let i=M(n);if(!i)throw Error("Manifest array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=M(r);if(!n)throw Error("Manifest object parse failed.");if(Array.isArray(n.items))return n.items.map((i)=>{let s=M(i);if(!s)throw Error("Manifest items entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=M(JSON.parse(i));if(!s)throw Error("Manifest JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=M(JSON.parse(r));if(!n)throw Error("Manifest JSONL entries must be objects.");return n})}async function kr(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 manifest URI: ${e}`);if(r)K(e,r);let[{S3Client:l,GetObjectCommand:_},{fromIni:o}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new l({region:a?.region,credentials:a?.profile?o({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new _({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function br(e,t,r){if(e.startsWith("s3://"))return kr(e,t,r);if(!ar(e))throw Error(`Manifest not found: ${e}`);return cr(e,"utf8")}function wr(e,t,r){let n=e.replace(/\r\n/g,`
|
|
233
|
+
`);if(!n.trim())return[];let i=[],s=0;while(s<n.length){let l=Math.min(n.length,s+t),_=l;if(l<n.length){let a=n.lastIndexOf(`
|
|
234
234
|
|
|
235
|
-
`,l),
|
|
235
|
+
`,l),c=n.lastIndexOf(". ",l),u=Math.max(a,c);if(u>s+Math.floor(t*0.5))_=u+(u===a?2:1)}let o=n.slice(s,_).trim();if(o)i.push({ordinal:i.length,text:o,startOffset:s,endOffset:_});if(_>=n.length)break;s=Math.max(0,_-r)}return i}function Rr(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Sr(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t);for(let n of r)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[n.id]);return e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]),r.length}function xr(e,t,r){let n=he("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
236
236
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
237
237
|
ON CONFLICT(uri) DO UPDATE SET
|
|
238
238
|
kind = excluded.kind,
|
|
239
239
|
title = excluded.title,
|
|
240
240
|
metadata_json = excluded.metadata_json,
|
|
241
241
|
acl_json = excluded.acl_json,
|
|
242
|
-
updated_at = excluded.updated_at`,[
|
|
242
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function vr(e,t,r,n){let i=he("rev",`${t}\x00${r.revision}`);e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
243
243
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
244
244
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
245
245
|
hash = excluded.hash,
|
|
246
246
|
extracted_text_uri = excluded.extracted_text_uri,
|
|
247
|
-
metadata_json = excluded.metadata_json`,[i,t,
|
|
248
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
247
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision);if(!s)throw Error(`Failed to upsert source revision: ${r.sourceRef}`);return s.id}function Or(e,t,r,n,i,s,l){if(!r.text||r.status.toLowerCase()==="deleted")return{chunksInserted:0,redactions:0};let _=ee(r.text,l);if(_.findings.length>0)te(e,{source_uri:r.sourceUri,findings:_.findings,metadata:{source_ref:r.sourceRef,revision:r.revision},created_at:n}),w(e,{event_type:"redaction",action:"source_text_redact",target_uri:r.sourceUri,decision:"redacted",metadata:{findings:_.findings.length,source_ref:r.sourceRef,revision:r.revision},created_at:n});let o=wr(_.text,i,s);for(let a of o){let c=he("chk",`${t}\x00${a.ordinal}\x00${a.text}`),u={source_ref:r.sourceRef,source_uri:r.sourceUri,hash:r.hash,status:r.status,path:E(r.raw.path)??null,mime:E(r.raw.mime)??E(r.raw.content_type)??null,size:dr(r.raw.size)??null};e.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
248
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[c,t,"source",a.ordinal,a.text,Rr(a.text),a.startOffset,a.endOffset,JSON.stringify(u),n]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[c,a.text,r.title??"",r.sourceUri])}return{chunksInserted:o.length,redactions:_.findings.length}}async function Be(e){let t=e.now??new Date;if(e.safetyPolicy)j(e.dbPath,e.safetyPolicy);U(e.dbPath);let r=await br(e.input,e.config,e.safetyPolicy),n=mr(r);return ye({dbPath:e.dbPath,items:n,sourceLabel:e.input,safetyPolicy:e.safetyPolicy,now:t,maxChunkChars:e.maxChunkChars,chunkOverlapChars:e.chunkOverlapChars})}async function ye(e){let t=(e.now??new Date).toISOString(),r=e.maxChunkChars??4000,n=e.chunkOverlapChars??200;if(r<500)throw Error("maxChunkChars must be at least 500.");if(n<0||n>=r)throw Error("chunkOverlapChars must be less than maxChunkChars.");if(e.safetyPolicy)j(e.dbPath,e.safetyPolicy);U(e.dbPath);let i=C(e.dbPath);try{return i.transaction(()=>{let l=new Set,_=new Set,o=0,a=0,c=0,u=0;w(i,{event_type:"source_read",action:e.readAction??(e.sourceLabel.startsWith("s3://")?"s3_manifest_read":"local_manifest_read"),target_uri:e.sourceLabel,decision:"allow",metadata:{items:e.items.length,read_only:!0},created_at:t});for(let f of e.items){let d=Tr(f,t),y=xr(i,d,t),T=vr(i,y,d,t);if(l.add(y),_.add(T),d.text||d.status.toLowerCase()==="deleted")a+=Sr(i,T);let p=Or(i,T,d,t,r,n,e.safetyPolicy);o+=p.chunksInserted,c+=p.redactions}return w(i,{event_type:"write",action:"knowledge_manifest_ingest",target_uri:e.dbPath,decision:"allow",metadata:{items:e.items.length,sources:l.size,revisions:_.size,chunks_inserted:o,redactions:c},created_at:t}),{path:e.sourceLabel,db_path:e.dbPath,items_seen:e.items.length,sources_upserted:l.size,revisions_upserted:_.size,chunks_inserted:o,chunks_deleted:a,redactions:c,skipped:u}})()}finally{i.close()}}import{createHash as Ur}from"crypto";import{existsSync as Pr,readFileSync as Kr}from"fs";import{basename as se}from"path";function ne(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function W(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function ze(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Nr(e,t){let r=e.mode;if(typeof r==="string"&&r!=="read_only")throw Error(`Source resolver denied ${t}. Permission mode is ${r}, expected read_only.`);let n=e.denied_purposes;if(Array.isArray(n)&&n.includes(t))throw Error(`Source resolver denied ${t}. Purpose is explicitly denied.`);let i=e.allowed_purposes;if(Array.isArray(i)&&i.length>0&&!i.includes(t))throw Error(`Source resolver denied ${t}. Allowed purposes: ${i.join(", ")}`)}function Lr(e,t,r){if(!t)return r;try{let n=N(e);if(n.kind==="open-files"&&n.entity==="file")return`${e}/revision/${encodeURIComponent(t.revision)}`}catch{return r}return r}function Ar(e,t,r){return e.query(`SELECT id, uri, kind, title, metadata_json, acl_json, updated_at
|
|
249
249
|
FROM sources
|
|
250
250
|
WHERE uri = ? OR uri = ?
|
|
251
251
|
ORDER BY CASE WHEN uri = ? THEN 0 ELSE 1 END
|
|
252
|
-
LIMIT 1`).get(t,
|
|
252
|
+
LIMIT 1`).get(t,r,t)??null}function Ir(e,t,r){if(r)return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
253
253
|
FROM source_revisions
|
|
254
254
|
WHERE source_id = ? AND revision = ?
|
|
255
|
-
LIMIT 1`).get(t,
|
|
255
|
+
LIMIT 1`).get(t,r)??null;return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
256
256
|
FROM source_revisions
|
|
257
257
|
WHERE source_id = ?
|
|
258
258
|
ORDER BY created_at DESC, revision DESC
|
|
259
|
-
LIMIT 1`).get(t)??null}function
|
|
259
|
+
LIMIT 1`).get(t)??null}function Dr(e,t){if(!t)return 0;return e.query("SELECT COUNT(*) AS n FROM chunks WHERE source_revision_id = ?").get(t)?.n??0}function Cr(e,t,r){if(!t||r<=0)return[];return e.query(`SELECT id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json
|
|
260
260
|
FROM chunks
|
|
261
261
|
WHERE source_revision_id = ?
|
|
262
262
|
ORDER BY ordinal ASC
|
|
263
|
-
LIMIT ?`).all(t,
|
|
263
|
+
LIMIT ?`).all(t,r)}async function ie(e){let t=e.purpose??"knowledge_answer",r=Math.max(0,Math.min(e.limit??10,100)),n=(e.now??new Date).toISOString(),i=N(e.sourceRef),s=Ue(e.sourceRef,i),l=Pe(e.sourceRef);if(e.safetyPolicy){if(!e.safetyPolicy.readOnlySourceAccess)throw Error("Safety policy denied source resolution.");j(e.dbPath,e.safetyPolicy)}U(e.dbPath);let _=C(e.dbPath);try{return _.transaction(()=>{let o=Ar(_,s,e.sourceRef);if(!o)return w(_,{event_type:"source_read",action:"open_files_resolve_missing",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:s},created_at:n}),{source_ref:e.sourceRef,source_uri:s,purpose:t,read_only:!0,resolved:!1,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:null,revision:null,content:{mime:null,size:null,hash:null,text_available:!1,chunks_total:0,chunks_returned:0,char_count_returned:0,extracted_text_ref:null,bytes_available:!1,bytes_exposed:!1},chunks:[],citations:[]};let a=ne(o.metadata_json),c=ne(o.acl_json);try{Nr(c,t)}catch(g){throw w(_,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"deny",metadata:{purpose:t,read_only:!0,source_uri:o.uri,error:g instanceof Error?g.message:String(g)},created_at:n}),g}let u=Ir(_,o.id,l),f=ne(u?.metadata_json),d=Dr(_,u?.id??null),y=Cr(_,u?.id??null,r),T=Lr(o.uri,u,e.sourceRef),p=y.map((g)=>{let O=ne(g.metadata_json),m={resolver:"open-files-read-only",mode:"local_catalog",purpose:t,read_only:!0,source_ref:W(O,["source_ref"])??T,source_uri:o.uri,source_revision_id:u?.id??null,revision:u?.revision??null,hash:u?.hash??W(O,["hash"]),chunk_id:g.id,start_offset:g.start_offset,end_offset:g.end_offset,resolved_at:n};return{id:g.id,kind:g.kind,ordinal:g.ordinal,text:g.text,token_count:g.token_count,start_offset:g.start_offset,end_offset:g.end_offset,metadata:O,evidence:m}}),L=p.map((g)=>({source_ref:g.evidence.source_ref,source_uri:o.uri,chunk_id:g.id,quote:g.text.slice(0,500),start_offset:g.start_offset,end_offset:g.end_offset,evidence:g.evidence}));w(_,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:o.uri,revision:u?.revision??null,chunks_returned:p.length,chunks_total:d},created_at:n});let k=W(a,["mime","content_type"])??W(f,["mime","content_type"]),R=ze(a,["size","size_bytes"])??ze(f,["size","size_bytes"]);return{source_ref:T,source_uri:o.uri,purpose:t,read_only:!0,resolved:!0,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:{id:o.id,uri:o.uri,kind:o.kind,title:o.title,metadata:a,permissions:c,updated_at:o.updated_at},revision:u?{id:u.id,revision:u.revision,hash:u.hash,extracted_text_uri:u.extracted_text_uri,metadata:f,created_at:u.created_at,reindex_required:f.reindex_required===!0}:null,content:{mime:k,size:R,hash:u?.hash??W(a,["hash","checksum","sha256"]),text_available:d>0,chunks_total:d,chunks_returned:p.length,char_count_returned:p.reduce((g,O)=>g+O.text.length,0),extracted_text_ref:u?.extracted_text_uri??W(f,["extracted_text_ref","extracted_text_uri"]),bytes_available:!1,bytes_exposed:!1},chunks:p,citations:L}})()}finally{_.close()}}function $(e){return`sha256:${Ur("sha256").update(e).digest("hex")}`}function jr(e){return e.replace(/<script[\s\S]*?<\/script>/gi," ").replace(/<style[\s\S]*?<\/style>/gi," ").replace(/<[^>]+>/g," ").replace(/ /g," ").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/\s+\n/g,`
|
|
264
264
|
`).replace(/\n\s+/g,`
|
|
265
|
-
`).replace(/[ \t]{2,}/g," ").trim()}async function
|
|
265
|
+
`).replace(/[ \t]{2,}/g," ").trim()}async function Fr(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 source URI: ${e}`);if(r)K(e,r);let[{S3Client:l,GetObjectCommand:_},{fromIni:o}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new l({region:a?.region,credentials:a?.profile?o({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new _({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function Xr(e,t){if(t)Z(t);let r=await fetch(e,{headers:{accept:"text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5","user-agent":"@hasna/knowledge source-ingest"}});if(!r.ok)throw Error(`Web source read failed ${r.status}: ${e}`);let n=r.headers.get("content-type"),i=await r.text();return{text:n?.includes("html")?jr(i):i,mime:n}}function oe(e){if(e.kind==="file")return se(e.path);if(e.kind==="s3")return se(e.key);if(e.kind==="web")return se(new URL(e.url).pathname)||e.url;return e.path?se(e.path):e.id}async function He(e,t,r){if(e.kind==="file"){if(!Pr(e.path))throw Error(`Source file not found: ${e.path}`);let n=Kr(e.path,"utf8");return{text:n,contentSource:"file",title:oe(e),mime:"text/plain",size:n.length,hash:$(n),revision:null,extractedTextRef:null,metadata:{path:e.path},permissions:{mode:"read_only"}}}if(e.kind==="s3"){let n=await Fr(e.uri,t,r);return{text:n,contentSource:"s3",title:oe(e),mime:"text/plain",size:n.length,hash:$(n),revision:null,extractedTextRef:null,metadata:{bucket:e.bucket,key:e.key},permissions:{mode:"read_only"}}}if(e.kind==="web"){let n=await Xr(e.url,r);return{text:n.text,contentSource:"web",title:oe(e),mime:n.mime,size:n.text.length,hash:$(n.text),revision:null,extractedTextRef:null,metadata:{url:e.url},permissions:{mode:"read_only"}}}throw Error(`Direct source reading is not available for ${e.uri}`)}async function Mr(e,t,r){if(e.startsWith("open-files://"))throw Error("Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.");let n=N(e);return{text:(await He(n,t,r)).text,contentSource:"extracted_text_ref"}}async function Wr(e){let t=await ie({dbPath:e.dbPath,sourceRef:e.sourceRef,purpose:e.purpose??"knowledge_index",limit:100,safetyPolicy:e.safetyPolicy,now:e.now});if(!t.resolved)throw Error("Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.");if(t.revision?.extracted_text_uri&&!t.content.text_available){let n=await Mr(t.revision.extracted_text_uri,e.config,e.safetyPolicy);return{text:n.text,contentSource:n.contentSource,title:t.source?.title??null,mime:t.content.mime,size:n.text.length,hash:t.revision.hash??$(n.text),revision:t.revision.revision,extractedTextRef:t.revision.extracted_text_uri,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}if(t.chunks.length===0)throw Error("Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.");let r=t.chunks.map((n)=>n.text).join(`
|
|
266
266
|
|
|
267
|
-
`);return{text:
|
|
267
|
+
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??$(r),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function $r(e,t,r,n){let i=r.hash??$(r.text),s={...r.metadata,source_ref:e,content_source:r.contentSource,read_only:!0},l={source_ref:e,name:r.title??oe(t),mime:r.mime??"text/plain",size:r.size??r.text.length,hash:i,revision:r.revision??i,status:"active",updated_at:new Date().toISOString(),permissions:{mode:"read_only",allowed_purposes:[n],...r.permissions},metadata:s,extracted_text_ref:r.extractedTextRef,extracted_text:r.text};if(t.kind==="open-files"){if(t.entity==="file")l.file_id=t.id;if(t.entity==="source")l.source_id=t.id,l.path=t.path}if(t.kind==="file")l.path=t.path;if(t.kind==="s3")l.path=t.key;if(t.kind==="web")l.url=t.url;return l}async function qe(e){let t=e.purpose??"knowledge_index",r=N(e.sourceRef),n=r.kind==="open-files"?await Wr(e):await He(r,e.config,e.safetyPolicy),i=$r(e.sourceRef,r,n,t);return{...await ye({dbPath:e.dbPath,items:[i],sourceLabel:e.sourceRef,readAction:"source_ref_ingest_read",safetyPolicy:e.safetyPolicy,now:e.now}),source_ref:e.sourceRef,content_source:n.contentSource,read_only:!0,hash:String(i.hash)}}var Ye={openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},Br={openai:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!0,reasoning:!0,embeddings:!0},anthropic:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!1,reasoning:!0,embeddings:!1},deepseek:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!1,native_web_search:!1,reasoning:!0,embeddings:!1}},zr={default:"openai:gpt-5.2",fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"};function Je(e){return e.providers??{}}function Hr(e,t){let r=Je(e)[t]??{};return{...Ye[t],...r}}function Ge(e){let t=Je(e);return{...zr,...t.default_model?{default:t.default_model}:{},...t.aliases??{}}}function Te(e){let[t,...r]=e.split(":"),n=r.join(":");if(t!=="openai"&&t!=="anthropic"&&t!=="deepseek")throw Error(`Unsupported AI provider: ${t}`);if(!n)throw Error(`Invalid model ref: ${e}. Expected provider:model.`);return{provider:t,model:n}}function me(e,t){return Ge(t)[e]??e}function ke(e){let t=Ge(e);return Object.entries(t).map(([r,n])=>{let i=Te(n);return{alias:r,model_ref:n,provider:i.provider,model:i.model,default:r==="default",capabilities:Br[i.provider]}})}function Ve(e,t=process.env){return Object.keys(Ye).map((r)=>{let n=Hr(e,r),i=Boolean(t[n.api_key_env]);return{provider:r,api_key_env:n.api_key_env,configured:i,source:i?"env":"missing",base_url:n.base_url??null,default_model:n.default_model}})}function Qe(e,t=process.env){return{default_model:me("default",e),providers:Ve(e,t),models:ke(e)}}function Ze(e,t,r=process.env){let n=Ve(t,r).find((i)=>i.provider===e);if(!n)throw Error(`Unsupported AI provider: ${e}`);if(!n.configured)throw Error(`Missing ${n.api_key_env} for ${e}. Set the env var to use this provider.`);return n}function qr(e){let t=String(e.getUTCFullYear()),r=String(e.getUTCMonth()+1).padStart(2,"0"),n=String(e.getUTCDate()).padStart(2,"0");return{year:t,month:r,day:n}}function Yr(){return`# Knowledge Agent Schema v1
|
|
268
268
|
|
|
269
269
|
## Source Rules
|
|
270
270
|
|
|
@@ -289,7 +289,7 @@ VALUES (3, datetime('now'));
|
|
|
289
289
|
## Lint Rules
|
|
290
290
|
|
|
291
291
|
- Flag stale pages, missing citations, contradictions, orphan pages, duplicate pages, and unresolved source refs.
|
|
292
|
-
`}function
|
|
292
|
+
`}function Jr(){return`# Knowledge Index
|
|
293
293
|
|
|
294
294
|
This is a compact orientation index for agents. It is not the full search index.
|
|
295
295
|
|
|
@@ -304,13 +304,13 @@ This is a compact orientation index for agents. It is not the full search index.
|
|
|
304
304
|
|
|
305
305
|
Raw source files are resolved through open-files. This app stores source refs,
|
|
306
306
|
citations, chunks, generated wiki artifacts, indexes, and run records.
|
|
307
|
-
`}function
|
|
307
|
+
`}function Gr(){return`# Wiki
|
|
308
308
|
|
|
309
309
|
Generated durable knowledge pages live here.
|
|
310
310
|
|
|
311
311
|
Pages should be concise, cited, and organized for both humans and agents.
|
|
312
|
-
`}async function
|
|
313
|
-
`,content_type:"application/x-ndjson"})];return await Promise.all(
|
|
312
|
+
`}async function et(e,t=new Date){let{year:r,month:n,day:i}=qr(t),s="schemas/v1.md",l="indexes/root.md",_="wiki/README.md",o=`logs/${r}/${n}/${i}.jsonl`,a={ts:t.toISOString(),event:"wiki_layout_initialized",schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md"},c=[e.put({key:"schemas/v1.md",body:Yr(),content_type:"text/markdown"}),e.put({key:"indexes/root.md",body:Jr(),content_type:"text/markdown"}),e.put({key:"wiki/README.md",body:Gr(),content_type:"text/markdown"}),e.put({key:o,body:`${JSON.stringify(a)}
|
|
313
|
+
`,content_type:"application/x-ndjson"})];return await Promise.all(c),{schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md",log_key:o,written:["schemas/v1.md","indexes/root.md","wiki/README.md",o]}}class tt{options;ensuredWorkspace;cachedConfig;constructor(e={}){this.options=e}get scope(){return this.options.scope??"global"}get workspace(){return this.ensuredWorkspace??Re(this.options.scope,this.options.cwd)}ensureWorkspace(){if(!this.ensuredWorkspace)this.ensuredWorkspace=we(this.workspace.home);return this.ensuredWorkspace}jsonStorePath(){return this.ensureWorkspace().jsonStorePath}config(){if(!this.cachedConfig){let e=this.ensureWorkspace();this.cachedConfig=Se(e.configPath)}return this.cachedConfig}safetyPolicy(){return Fe(this.config(),this.ensureWorkspace())}artifactStore(){return De(this.config(),this.ensureWorkspace())}paths(){let e=this.ensureWorkspace();return{ok:!0,scope:this.scope,home:e.home,config_path:e.configPath,json_store_path:e.jsonStorePath,knowledge_db_path:e.knowledgeDbPath,artifacts_dir:e.artifactsDir,indexes_dir:e.indexesDir,logs_dir:e.logsDir,runs_dir:e.runsDir,schemas_dir:e.schemasDir,wiki_dir:e.wikiDir,config:this.config(),message:e.home}}initDb(){return U(this.ensureWorkspace().knowledgeDbPath)}dbStats(){let e=this.ensureWorkspace();return U(e.knowledgeDbPath),Ne(e.knowledgeDbPath)}async initWiki(){return et(this.artifactStore())}async ingestManifest(e){let t=this.ensureWorkspace();return Be({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}async ingestSource(e,t){let r=this.ensureWorkspace();return qe({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t,config:this.config(),safetyPolicy:this.safetyPolicy()})}async resolveSource(e,t={}){let r=this.ensureWorkspace();return ie({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t.purpose,limit:t.limit,safetyPolicy:this.safetyPolicy()})}async consumeOutbox(e){let t=this.ensureWorkspace();return $e({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}providerStatus(e=process.env){return Qe(this.config(),e)}modelRegistry(){return ke(this.config())}}function rt(e={}){return new tt(e)}var H={name:"@hasna/knowledge",version:"0.2.11",description:"Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",type:"module",bin:{"open-knowledge":"bin/open-knowledge.js","open-knowledge-mcp":"bin/open-knowledge-mcp.js"},files:["bin","src","docs","LICENSE","README.md"],scripts:{test:"bun test","test:cli":"bun test tests/cli.test.ts",build:"bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",prepublishOnly:"bun run build",postinstall:"bun run build"},keywords:["knowledge","cli","agents","json","notes","local","store"],license:"Apache-2.0",publishConfig:{registry:"https://registry.npmjs.org",access:"public"},repository:{type:"git",url:"git+https://github.com/hasna/knowledge.git"},bugs:{url:"https://github.com/hasna/knowledge/issues"},author:"Hasna Inc. <hasna@example.com>",engines:{bun:">=1.0",node:">=18"},dependencies:{"@aws-sdk/client-s3":"^3.1063.0","@aws-sdk/credential-providers":"^3.1063.0","@ai-sdk/anthropic":"^3.0.81","@ai-sdk/deepseek":"^2.0.35","@ai-sdk/openai":"^3.0.68","@modelcontextprotocol/sdk":"^1.29.0",ai:"^6.0.197",zod:"^4.3.6"},devDependencies:{"@types/bun":"^1.3.14"}};var nt={debug:0,info:1,warn:2,error:3},Qr=()=>{if(process.env.DEBUG)return"debug";if(process.env.LOG_LEVEL==="debug")return"debug";if(process.env.LOG_LEVEL==="warn")return"warn";if(process.env.LOG_LEVEL==="error")return"error";return"info"};function F(e,t,r){if(nt[e]<nt[Qr()])return;let n={debug:"[DEBUG]",info:"[INFO]",warn:"[WARN]",error:"[ERROR]"}[e],i=r?`${n} ${t} ${JSON.stringify(r)}`:`${n} ${t}`;if(e==="error")console.error(i);else console.error(i)}var Zr=["add","list","get","delete","update","archive","restore","upsert","untag","export","prune","dedupe","stats","paths","db","wiki","source","ingest","reindex","providers","safety","help"],it={ls:"list",rm:"delete",edit:"update",unarchive:"restore"};function en(e){let t=[],r={};for(let n=0;n<e.length;n+=1){let i=e[n];if(!i.startsWith("-")){t.push(i);continue}switch(i){case"--json":r.json=!0;break;case"--yes":case"-y":r.yes=!0;break;case"--help":case"-h":r.help=!0;break;case"--version":case"-v":r.version=!0;break;case"--desc":r.desc=!0;break;case"--page":case"-p":r.page=Number(e[n+1]),n+=1;break;case"--limit":case"-l":r.limit=Number(e[n+1]),n+=1;break;case"--search":case"-s":r.search=e[n+1],n+=1;break;case"--sort":r.sort=e[n+1],n+=1;break;case"--id":r.id=e[n+1],n+=1;break;case"--store":r.store=e[n+1],n+=1;break;case"--title":r.title=e[n+1],n+=1;break;case"--content":r.content=e[n+1],n+=1;break;case"--url":r.url=e[n+1],n+=1;break;case"--tag":case"-t":r.tag=e[n+1],n+=1;break;case"--format":r.format=e[n+1],n+=1;break;case"--completions":r.completions=e[n+1],n+=1;break;case"--purpose":r.purpose=e[n+1],n+=1;break;case"--no-color":r.noColor=!0;break;case"--scope":r.scope=e[n+1],n+=1;break;case"--older-than":r.olderThan=Number(e[n+1]),n+=1;break;case"--empty":r.empty=!0;break;case"--archived":r.archived=!0;break;case"--include-archived":r.includeArchived=!0;break;default:throw Error(`Unknown flag: ${i}. Run 'open-knowledge --help' for valid options.`)}}return{positional:t,flags:r}}function tn(e){if(!e)return"";return it[e]??e}function rn(e,t){let r=Array.from({length:e.length+1},()=>Array(t.length+1).fill(0));for(let n=0;n<=e.length;n+=1)r[n][0]=n;for(let n=0;n<=t.length;n+=1)r[0][n]=n;for(let n=1;n<=e.length;n+=1)for(let i=1;i<=t.length;i+=1){let s=e[n-1]===t[i-1]?0:1;r[n][i]=Math.min(r[n-1][i]+1,r[n][i-1]+1,r[n-1][i-1]+s)}return r[e.length][t.length]}function nn(e){if(!e)return"";let t=[...Zr,...Object.keys(it)],r="",n=Number.POSITIVE_INFINITY;for(let i of t){let s=rn(e,i);if(s<n)n=s,r=i}return n<=3?r:""}function sn(){console.log(`open-knowledge - local agent knowledge store
|
|
314
314
|
|
|
315
315
|
Usage:
|
|
316
316
|
open-knowledge <command> [options]
|
|
@@ -336,6 +336,7 @@ Commands:
|
|
|
336
336
|
ingest manifest <file|s3://> Ingest an open-files manifest into knowledge.db
|
|
337
337
|
ingest source <source-ref> Ingest a read-only source ref into knowledge.db
|
|
338
338
|
reindex outbox <file|s3://> Consume open-files change events and invalidate chunks
|
|
339
|
+
providers status|models|check Inspect AI SDK provider config and credentials
|
|
339
340
|
safety status|check|approve|audit|redact
|
|
340
341
|
help [command] Show help
|
|
341
342
|
|
|
@@ -379,5 +380,5 @@ Export Options:
|
|
|
379
380
|
|
|
380
381
|
Prune Options:
|
|
381
382
|
--older-than <days> Remove items older than N days
|
|
382
|
-
--empty Remove items with empty content`)}function
|
|
383
|
-
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(o==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let r=Bn(t[0]);if(!r||n.help||r==="help"){Yn(t[1]);return}let i=qe({scope:n.scope}),s=n.store;if(!s)if(n.scope==="project"||n.scope==="local")s=i.jsonStorePath();else s=le();if(r==="paths"){m(i.paths(),n.json);return}if(r==="db"){let o=t[1]??"init";if(o!=="init"&&o!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(o==="init"){let a=i.initDb();m({ok:!0,...a,message:`Initialized ${a.path}`},n.json);return}let c=i.dbStats();m({ok:!0,path:i.workspace.knowledgeDbPath,...c,message:`knowledge.db schema v${c.schema_version}`},n.json);return}if(r==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let c=await i.initWiki();m({ok:!0,...c,message:`Initialized wiki layout in ${i.workspace.home}`},n.json);return}if(r==="safety"){let o=t[1]??"status",c=i.ensureWorkspace(),a=i.safetyPolicy();i.initDb();let u=C(c.knowledgeDbPath);try{if(o==="status"){m({ok:!0,mode:a.mode,workspace:c.home,allow_write_roots:a.allowWriteRoots,read_only_source_access:a.readOnlySourceAccess,network:a.network,redaction:a.redaction,approvals:a.approvals,message:`Safety policy: ${a.mode}`},n.json);return}if(o==="check"){let f=t[2]??"generated_write",d=t[3]??null,h;try{if(f==="web_search")Z(a),h={action:f,target_uri:d,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!d)throw Error("safety check s3_read requires an s3:// target.");X(d,a),h={action:f,target_uri:d,approval_required:!1,approved:!0,decision:"allow"}}else h=Fe(u,a,f,d);S(u,{event_type:"safety_check",action:f,target_uri:d,decision:h.decision==="allow"?"allow":"requires_approval",metadata:h}),m({ok:!0,...h,message:`Safety check ${h.decision}`},n.json);return}catch(T){throw S(u,{event_type:"safety_check",action:f,target_uri:d,decision:"deny",metadata:{error:T instanceof Error?T.message:String(T)}}),T}}if(o==="approve"){let f=t[2]??"generated_write",d=t[3]??null,h=Xe(u,{action:f,target_uri:d,reason:"local-cli approval",metadata:{scope:n.scope??"global"}});S(u,{event_type:"approval",action:f,target_uri:d,decision:"allow",metadata:{approval_id:h.id}}),m({ok:!0,...h,action:f,target_uri:d,message:`Approved ${f}`},n.json);return}if(o==="audit"){let f=u.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((d)=>({id:d.id,event_type:d.event_type,action:d.action,target_uri:d.target_uri,decision:d.decision,metadata:JSON.parse(d.metadata_json),created_at:d.created_at}));m({ok:!0,events:f,message:`${f.length} audit event(s)`},n.json);return}if(o==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let d=ee(f,a);if(d.findings.length>0)te(u,{source_uri:"safety://redact",findings:d.findings,metadata:{command:"safety redact"}});S(u,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:d.findings.length>0?"redacted":"allow",metadata:{findings:d.findings.length}}),m({ok:!0,text:d.text,findings:d.findings,message:`Redacted ${d.findings.length} finding(s)`},n.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{u.close()}}if(r==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let c=t[2];if(!c)throw Error("Usage: open-knowledge source resolve <source-ref>");let a=await i.resolveSource(c,{purpose:n.purpose,limit:n.limit});m({ok:!0,...a,message:a.resolved?`Resolved ${a.source_ref} (${a.content.chunks_returned}/${a.content.chunks_total} chunks)`:`Source not indexed: ${c}`},n.json);return}if(r==="ingest"){let o=t[1]??"";if(o==="manifest"){let c=t[2];if(!c)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let a=await i.ingestManifest(c);m({ok:!0,...a,message:`Ingested ${a.items_seen} manifest item(s)`},n.json);return}if(o==="source"){let c=t[2];if(!c)throw Error("Usage: open-knowledge ingest source <source-ref>");let a=await i.ingestSource(c,n.purpose);m({ok:!0,...a,message:`Ingested source ${a.source_ref} (${a.chunks_inserted} chunks)`},n.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(r==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let c=t[2];if(!c)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let a=await i.consumeOutbox(c);m({ok:!0,...a,message:`Consumed ${a.events_seen} outbox event(s)`},n.json);return}if(fe(s),r==="add"){let o=t[1],c=t[2];if(!o||!c)throw Error("Usage: open-knowledge add <title> <content>");O(s,()=>{let a=x(s),u={id:_e(),title:o,content:c,url:n.url??null,tags:n.tag?[n.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};a.items.push(u),D(s,a),K("info","Item added",{id:u.id,title:u.title}),m({ok:!0,item:u,message:`Added ${u.id}`},n.json)});return}if(r==="list"){if(n.format!==void 0&&n.format!=="table"&&n.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");O(s,()=>{let o=x(s),c=Number.isFinite(n.page)&&n.page>0?n.page:1,a=Number.isFinite(n.limit)&&n.limit>0?n.limit:20,u=n.search?String(n.search).toLowerCase():"",f=n.tag?String(n.tag).toLowerCase():"",d=n.format==="table"||!n.json&&!n.format&&Jn(n),h=n.json||n.format==="json",T=o.items;if(n.archived)T=T.filter((y)=>y.archived===!0);else if(!n.includeArchived)T=T.filter((y)=>!y.archived);if(u)T=T.filter((y)=>y.title.toLowerCase().includes(u)||y.content.toLowerCase().includes(u));if(f)T=T.filter((y)=>y.tags&&y.tags.map((ae)=>ae.toLowerCase()).includes(f));let{sorted:p,sort:L,direction:b}=Gn(T,n),R=(c-1)*a,g=p.slice(R,R+a),N=Math.max(1,Math.ceil(p.length/a));if(h){m({ok:!0,page:c,limit:a,total:p.length,total_pages:N,sort:L,direction:b,items:g},!0);return}if(g.length===0){m(`No items found (search=${u||"none"}, tag=${f||"none"})`,!1);return}if(d){let y=(j)=>j,ae=`${y("ID")} ${y("TITLE")} ${y("CREATED")} ${y("URL")} ${y("TAGS")}`;console.log(ae);for(let j of g)console.log(`${j.id} ${y(j.title)} ${j.created_at} ${j.url?y(j.url):""} ${j.tags?.length?y(`[${j.tags.join(", ")}]`):""}`);console.log(`Page ${c}/${N} | showing ${g.length} of ${p.length} | sort=${L} ${b} | search=${u||"none"} | tag=${f||"none"}`)}else{for(let y of g)console.log(`${y.id} ${y.title} ${y.created_at}${y.url?` ${y.url}`:""}${y.tags?.length?` [${y.tags.join(", ")}]`:""}`);console.log(`Page ${c}/${N} | showing ${g.length} of ${p.length} | sort=${L} ${b} | search=${u||"none"} | tag=${f||"none"}`)}});return}if(r==="get"){q(n),O(s,()=>{let c=x(s).items.find((a)=>a.id===n.id||a.short_id===n.id);if(!c)throw Error(`Item not found: ${n.id}`);m({ok:!0,item:c,message:`${c.id}: ${c.title}`},n.json)});return}if(r==="update"){q(n),O(s,()=>{let o=x(s),c=o.items.findIndex((u)=>u.id===n.id||u.short_id===n.id);if(c===-1)throw Error(`Item not found: ${n.id}`);let a=o.items[c];if(n.title!==void 0)a.title=n.title;if(n.content!==void 0)a.content=n.content;if(n.url!==void 0)a.url=n.url;if(n.tag!==void 0){if(a.tags=a.tags||[],!a.tags.map((u)=>u.toLowerCase()).includes(n.tag.toLowerCase()))a.tags.push(n.tag)}a.updated_at=new Date().toISOString(),o.items[c]=a,D(s,o),m({ok:!0,item:a,message:`Updated ${a.id}`},n.json)});return}if(r==="archive"||r==="restore"){q(n),O(s,()=>{let o=x(s),c=o.items.findIndex((u)=>u.id===n.id||u.short_id===n.id);if(c===-1)throw Error(`Item not found: ${n.id}`);let a=o.items[c];a.archived=r==="archive",a.updated_at=new Date().toISOString(),o.items[c]=a,D(s,o),m({ok:!0,item:a,message:`${r==="archive"?"Archived":"Restored"} ${a.id}`},n.json)});return}if(r==="untag"){if(q(n),!n.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");O(s,()=>{let o=x(s),c=o.items.findIndex((f)=>f.id===n.id||f.short_id===n.id);if(c===-1)throw Error(`Item not found: ${n.id}`);let a=o.items[c],u=a.tags?.length??0;a.tags=(a.tags??[]).filter((f)=>f.toLowerCase()!==n.tag.toLowerCase()),a.updated_at=new Date().toISOString(),o.items[c]=a,D(s,o),m({ok:!0,item:a,removed:u-a.tags.length,message:`Removed tag from ${a.id}`},n.json)});return}if(r==="upsert"){let o=n.title??t[1],c=n.content??t[2];O(s,()=>{let a=x(s),u=n.id?a.items.findIndex((h)=>h.id===n.id||h.short_id===n.id):-1,f=new Date().toISOString();if(u===-1){if(!o||!c)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let h=n.id??_e(),T={id:h,short_id:ke(h),title:o,content:c,url:n.url??null,tags:n.tag?[n.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};a.items.push(T),D(s,a),m({ok:!0,created:!0,item:T,message:`Upserted ${T.id}`},n.json);return}let d=a.items[u];if(o!==void 0)d.title=o;if(c!==void 0)d.content=c;if(n.url!==void 0)d.url=n.url;if(n.tag!==void 0){if(d.tags=d.tags||[],!d.tags.map((h)=>h.toLowerCase()).includes(n.tag.toLowerCase()))d.tags.push(n.tag)}d.updated_at=f,a.items[u]=d,D(s,a),m({ok:!0,created:!1,item:d,message:`Upserted ${d.id}`},n.json)});return}if(r==="delete"){if(q(n),!n.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");O(s,()=>{let o=x(s),c=o.items.length;o.items=o.items.filter((u)=>u.id!==n.id&&u.short_id!==n.id);let a=c!==o.items.length;if(D(s,o),!a)throw Error(`Item not found: ${n.id}`);K("info","Item deleted",{id:n.id}),m({ok:!0,deleted_id:n.id,message:`Deleted ${n.id}`},n.json)});return}if(r==="export"){let o=n.format??"json";if(o!=="json"&&o!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");O(s,()=>{let c=x(s);if(o==="jsonl")for(let a of c.items)console.log(JSON.stringify(a));else m({ok:!0,items:c.items},n.json)});return}if(r==="prune"){if(!n.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");O(s,()=>{let o=x(s),c=o.items.length;if(n.olderThan!==void 0){let u=new Date;u.setDate(u.getDate()-n.olderThan),o.items=o.items.filter((f)=>new Date(f.created_at)>=u)}if(n.empty)o.items=o.items.filter((u)=>u.content.trim().length>0);let a=c-o.items.length;D(s,o),K("info","Prune completed",{pruned:a,remaining:o.items.length}),m({ok:!0,pruned:a,remaining:o.items.length,message:`Pruned ${a} item(s)`},n.json)});return}if(r==="dedupe"){if(!n.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");O(s,()=>{let o=x(s),c=new Set,a=o.items.length;o.items=o.items.filter((f)=>{let d=`${f.title}\x00${f.content}`;if(c.has(d))return!1;return c.add(d),!0});let u=a-o.items.length;D(s,o),K("info","Dedupe completed",{removed:u,remaining:o.items.length}),m({ok:!0,removed:u,remaining:o.items.length,message:`Dedupe removed ${u} duplicate(s)`},n.json)});return}if(r==="stats"){O(s,()=>{let o=x(s),c=o.items.filter((b)=>!b.archived),a=c.length,u=o.items.length-a,f=c.filter((b)=>b.url).length,d=c.filter((b)=>b.tags&&b.tags.length>0).length,h=a>0?c.map((b)=>b.created_at).sort()[0]:null,T=a>0?c.map((b)=>b.created_at).sort()[a-1]:null,p={};for(let b of c)for(let R of b.tags||[])p[R]=(p[R]||0)+1;let L=Object.entries(p).sort((b,R)=>R[1]-b[1]).slice(0,5).map(([b,R])=>({tag:b,count:R}));m({ok:!0,total:a,archived:u,with_url:f,with_tags:d,oldest:h,newest:T,top_tags:L,message:`${a} items | ${f} with URL | ${d} with tags`},n.json)});return}let l=Hn(t[0]),_=l?` Did you mean '${l}'?`:"";throw K("warn","Unknown command",{input:t[0],suggestion:l}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)Vn(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);K("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{Hn as suggestCommand,Gn as sortItems,Vn as run,$n as parseArgs};
|
|
383
|
+
--empty Remove items with empty content`)}function on(e){if(e==="add"){console.log("Usage: open-knowledge add <title> <content> [--url <url>] [-t <tag>] [--json]");return}if(e==="list"||e==="ls"){console.log("Usage: open-knowledge list|ls [--format table|json] [-p <page>] [-l <limit>] [-s <search>] [-t <tag>] [--sort created|title] [--desc] [--json]");return}if(e==="get"){console.log("Usage: open-knowledge get --id <id> [--json]");return}if(e==="update"||e==="edit"){console.log("Usage: open-knowledge update|edit --id <id> [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="archive"){console.log("Usage: open-knowledge archive --id <id> [--json]");return}if(e==="restore"||e==="unarchive"){console.log("Usage: open-knowledge restore|unarchive --id <id> [--json]");return}if(e==="upsert"){console.log("Usage: open-knowledge upsert [title] [content] [--id <id>] [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="untag"){console.log("Usage: open-knowledge untag --id <id> -t <tag> [--json]");return}if(e==="delete"||e==="rm"){console.log("Usage: open-knowledge delete|rm --id <id> -y [--json]");return}if(e==="export"){console.log("Usage: open-knowledge export [--format jsonl] [--json]");return}if(e==="prune"){console.log("Usage: open-knowledge prune --yes [--older-than <days>] [--empty] [--json]");return}if(e==="dedupe"){console.log("Usage: open-knowledge dedupe --yes [--json]");return}if(e==="stats"){console.log("Usage: open-knowledge stats [--json]");return}if(e==="paths"){console.log("Usage: open-knowledge paths [--scope local|global|project] [--json]");return}if(e==="db"){console.log("Usage: open-knowledge db init|stats [--scope local|global|project] [--json]");return}if(e==="wiki"){console.log("Usage: open-knowledge wiki init [--scope local|global|project] [--json]");return}if(e==="source"){console.log("Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]");return}if(e==="ingest"){console.log("Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]");return}if(e==="reindex"){console.log("Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]");return}if(e==="providers"){console.log("Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]");return}if(e==="safety"){console.log("Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]");return}sn()}function an(e){if(e.noColor||process.env.NO_COLOR)return!1;if(process.env.FORCE_COLOR)return!0;return process.stdout.isTTY===!0}function h(e,t,r){if(t){console.log(JSON.stringify(e,null,2));return}if(typeof e==="string"){console.log(e);return}console.log(e.message??JSON.stringify(e,null,2))}function q(e){if(!e.id)throw Error("Missing required --id. Example: open-knowledge get --id <id>")}function cn(e,t){let r=t.sort??"created";if(r!=="created"&&r!=="title")throw Error("Invalid --sort value. Use 'created' or 'title'.");let n=[...e].sort((i,s)=>{if(r==="title")return i.title.localeCompare(s.title);return i.created_at.localeCompare(s.created_at)});if(t.desc)n.reverse();return{sorted:n,sort:r,direction:t.desc?"desc":"asc"}}async function un(e){let{positional:t,flags:r}=en(e);if(F("debug","CLI invoked",{command:t[0],flags:{json:r.json,store:r.store}}),r.version){console.log(r.json?JSON.stringify({name:H.name,version:H.version},null,2):`${H.name} ${H.version}`);return}if(r.completions){let o=r.completions;if(o==="bash")console.log('_open_knowledge() { local cur; cur="${COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge');else if(o==="zsh")console.log(`#compdef open-knowledge
|
|
384
|
+
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(o==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=tn(t[0]);if(!n||r.help||n==="help"){on(t[1]);return}let i=rt({scope:r.scope}),s=r.store;if(!s)if(r.scope==="project"||r.scope==="local")s=i.jsonStorePath();else s=le();if(n==="paths"){h(i.paths(),r.json);return}if(n==="db"){let o=t[1]??"init";if(o!=="init"&&o!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(o==="init"){let c=i.initDb();h({ok:!0,...c,message:`Initialized ${c.path}`},r.json);return}let a=i.dbStats();h({ok:!0,path:i.workspace.knowledgeDbPath,...a,message:`knowledge.db schema v${a.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let a=await i.initWiki();h({ok:!0,...a,message:`Initialized wiki layout in ${i.workspace.home}`},r.json);return}if(n==="safety"){let o=t[1]??"status",a=i.ensureWorkspace(),c=i.safetyPolicy();i.initDb();let u=C(a.knowledgeDbPath);try{if(o==="status"){h({ok:!0,mode:c.mode,workspace:a.home,allow_write_roots:c.allowWriteRoots,read_only_source_access:c.readOnlySourceAccess,network:c.network,redaction:c.redaction,approvals:c.approvals,message:`Safety policy: ${c.mode}`},r.json);return}if(o==="check"){let f=t[2]??"generated_write",d=t[3]??null,y;try{if(f==="web_search")Z(c),y={action:f,target_uri:d,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!d)throw Error("safety check s3_read requires an s3:// target.");K(d,c),y={action:f,target_uri:d,approval_required:!1,approved:!0,decision:"allow"}}else y=Me(u,c,f,d);w(u,{event_type:"safety_check",action:f,target_uri:d,decision:y.decision==="allow"?"allow":"requires_approval",metadata:y}),h({ok:!0,...y,message:`Safety check ${y.decision}`},r.json);return}catch(T){throw w(u,{event_type:"safety_check",action:f,target_uri:d,decision:"deny",metadata:{error:T instanceof Error?T.message:String(T)}}),T}}if(o==="approve"){let f=t[2]??"generated_write",d=t[3]??null,y=Xe(u,{action:f,target_uri:d,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});w(u,{event_type:"approval",action:f,target_uri:d,decision:"allow",metadata:{approval_id:y.id}}),h({ok:!0,...y,action:f,target_uri:d,message:`Approved ${f}`},r.json);return}if(o==="audit"){let f=u.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((d)=>({id:d.id,event_type:d.event_type,action:d.action,target_uri:d.target_uri,decision:d.decision,metadata:JSON.parse(d.metadata_json),created_at:d.created_at}));h({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(o==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let d=ee(f,c);if(d.findings.length>0)te(u,{source_uri:"safety://redact",findings:d.findings,metadata:{command:"safety redact"}});w(u,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:d.findings.length>0?"redacted":"allow",metadata:{findings:d.findings.length}}),h({ok:!0,text:d.text,findings:d.findings,message:`Redacted ${d.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{u.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let a=t[2];if(!a)throw Error("Usage: open-knowledge source resolve <source-ref>");let c=await i.resolveSource(a,{purpose:r.purpose,limit:r.limit});h({ok:!0,...c,message:c.resolved?`Resolved ${c.source_ref} (${c.content.chunks_returned}/${c.content.chunks_total} chunks)`:`Source not indexed: ${a}`},r.json);return}if(n==="ingest"){let o=t[1]??"";if(o==="manifest"){let a=t[2];if(!a)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let c=await i.ingestManifest(a);h({ok:!0,...c,message:`Ingested ${c.items_seen} manifest item(s)`},r.json);return}if(o==="source"){let a=t[2];if(!a)throw Error("Usage: open-knowledge ingest source <source-ref>");let c=await i.ingestSource(a,r.purpose);h({ok:!0,...c,message:`Ingested source ${c.source_ref} (${c.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let a=t[2];if(!a)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let c=await i.consumeOutbox(a);h({ok:!0,...c,message:`Consumed ${c.events_seen} outbox event(s)`},r.json);return}if(n==="providers"){let o=t[1]??"status";if(o==="status"){let a=i.providerStatus(),c=a.providers.filter((u)=>u.configured).length;h({ok:!0,...a,message:`${c}/${a.providers.length} provider credential(s) configured`},r.json);return}if(o==="models"){let a=i.modelRegistry();h({ok:!0,models:a,message:`${a.length} model alias(es)`},r.json);return}if(o==="check"){let a=t[2]??"default",c=me(a,i.config()),u=Te(c),f=Ze(u.provider,i.config());h({ok:!0,target:a,model_ref:c,provider:u.provider,model:u.model,credential:f,message:`${u.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(fe(s),n==="add"){let o=t[1],a=t[2];if(!o||!a)throw Error("Usage: open-knowledge add <title> <content>");v(s,()=>{let c=x(s),u={id:_e(),title:o,content:a,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};c.items.push(u),D(s,c),F("info","Item added",{id:u.id,title:u.title}),h({ok:!0,item:u,message:`Added ${u.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");v(s,()=>{let o=x(s),a=Number.isFinite(r.page)&&r.page>0?r.page:1,c=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,u=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",d=r.format==="table"||!r.json&&!r.format&&an(r),y=r.json||r.format==="json",T=o.items;if(r.archived)T=T.filter((m)=>m.archived===!0);else if(!r.includeArchived)T=T.filter((m)=>!m.archived);if(u)T=T.filter((m)=>m.title.toLowerCase().includes(u)||m.content.toLowerCase().includes(u));if(f)T=T.filter((m)=>m.tags&&m.tags.map((ae)=>ae.toLowerCase()).includes(f));let{sorted:p,sort:L,direction:k}=cn(T,r),R=(a-1)*c,g=p.slice(R,R+c),O=Math.max(1,Math.ceil(p.length/c));if(y){h({ok:!0,page:a,limit:c,total:p.length,total_pages:O,sort:L,direction:k,items:g},!0);return}if(g.length===0){h(`No items found (search=${u||"none"}, tag=${f||"none"})`,!1);return}if(d){let m=(P)=>P,ae=`${m("ID")} ${m("TITLE")} ${m("CREATED")} ${m("URL")} ${m("TAGS")}`;console.log(ae);for(let P of g)console.log(`${P.id} ${m(P.title)} ${P.created_at} ${P.url?m(P.url):""} ${P.tags?.length?m(`[${P.tags.join(", ")}]`):""}`);console.log(`Page ${a}/${O} | showing ${g.length} of ${p.length} | sort=${L} ${k} | search=${u||"none"} | tag=${f||"none"}`)}else{for(let m of g)console.log(`${m.id} ${m.title} ${m.created_at}${m.url?` ${m.url}`:""}${m.tags?.length?` [${m.tags.join(", ")}]`:""}`);console.log(`Page ${a}/${O} | showing ${g.length} of ${p.length} | sort=${L} ${k} | search=${u||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){q(r),v(s,()=>{let a=x(s).items.find((c)=>c.id===r.id||c.short_id===r.id);if(!a)throw Error(`Item not found: ${r.id}`);h({ok:!0,item:a,message:`${a.id}: ${a.title}`},r.json)});return}if(n==="update"){q(r),v(s,()=>{let o=x(s),a=o.items.findIndex((u)=>u.id===r.id||u.short_id===r.id);if(a===-1)throw Error(`Item not found: ${r.id}`);let c=o.items[a];if(r.title!==void 0)c.title=r.title;if(r.content!==void 0)c.content=r.content;if(r.url!==void 0)c.url=r.url;if(r.tag!==void 0){if(c.tags=c.tags||[],!c.tags.map((u)=>u.toLowerCase()).includes(r.tag.toLowerCase()))c.tags.push(r.tag)}c.updated_at=new Date().toISOString(),o.items[a]=c,D(s,o),h({ok:!0,item:c,message:`Updated ${c.id}`},r.json)});return}if(n==="archive"||n==="restore"){q(r),v(s,()=>{let o=x(s),a=o.items.findIndex((u)=>u.id===r.id||u.short_id===r.id);if(a===-1)throw Error(`Item not found: ${r.id}`);let c=o.items[a];c.archived=n==="archive",c.updated_at=new Date().toISOString(),o.items[a]=c,D(s,o),h({ok:!0,item:c,message:`${n==="archive"?"Archived":"Restored"} ${c.id}`},r.json)});return}if(n==="untag"){if(q(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");v(s,()=>{let o=x(s),a=o.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(a===-1)throw Error(`Item not found: ${r.id}`);let c=o.items[a],u=c.tags?.length??0;c.tags=(c.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),c.updated_at=new Date().toISOString(),o.items[a]=c,D(s,o),h({ok:!0,item:c,removed:u-c.tags.length,message:`Removed tag from ${c.id}`},r.json)});return}if(n==="upsert"){let o=r.title??t[1],a=r.content??t[2];v(s,()=>{let c=x(s),u=r.id?c.items.findIndex((y)=>y.id===r.id||y.short_id===r.id):-1,f=new Date().toISOString();if(u===-1){if(!o||!a)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let y=r.id??_e(),T={id:y,short_id:Oe(y),title:o,content:a,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};c.items.push(T),D(s,c),h({ok:!0,created:!0,item:T,message:`Upserted ${T.id}`},r.json);return}let d=c.items[u];if(o!==void 0)d.title=o;if(a!==void 0)d.content=a;if(r.url!==void 0)d.url=r.url;if(r.tag!==void 0){if(d.tags=d.tags||[],!d.tags.map((y)=>y.toLowerCase()).includes(r.tag.toLowerCase()))d.tags.push(r.tag)}d.updated_at=f,c.items[u]=d,D(s,c),h({ok:!0,created:!1,item:d,message:`Upserted ${d.id}`},r.json)});return}if(n==="delete"){if(q(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");v(s,()=>{let o=x(s),a=o.items.length;o.items=o.items.filter((u)=>u.id!==r.id&&u.short_id!==r.id);let c=a!==o.items.length;if(D(s,o),!c)throw Error(`Item not found: ${r.id}`);F("info","Item deleted",{id:r.id}),h({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let o=r.format??"json";if(o!=="json"&&o!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");v(s,()=>{let a=x(s);if(o==="jsonl")for(let c of a.items)console.log(JSON.stringify(c));else h({ok:!0,items:a.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");v(s,()=>{let o=x(s),a=o.items.length;if(r.olderThan!==void 0){let u=new Date;u.setDate(u.getDate()-r.olderThan),o.items=o.items.filter((f)=>new Date(f.created_at)>=u)}if(r.empty)o.items=o.items.filter((u)=>u.content.trim().length>0);let c=a-o.items.length;D(s,o),F("info","Prune completed",{pruned:c,remaining:o.items.length}),h({ok:!0,pruned:c,remaining:o.items.length,message:`Pruned ${c} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");v(s,()=>{let o=x(s),a=new Set,c=o.items.length;o.items=o.items.filter((f)=>{let d=`${f.title}\x00${f.content}`;if(a.has(d))return!1;return a.add(d),!0});let u=c-o.items.length;D(s,o),F("info","Dedupe completed",{removed:u,remaining:o.items.length}),h({ok:!0,removed:u,remaining:o.items.length,message:`Dedupe removed ${u} duplicate(s)`},r.json)});return}if(n==="stats"){v(s,()=>{let o=x(s),a=o.items.filter((k)=>!k.archived),c=a.length,u=o.items.length-c,f=a.filter((k)=>k.url).length,d=a.filter((k)=>k.tags&&k.tags.length>0).length,y=c>0?a.map((k)=>k.created_at).sort()[0]:null,T=c>0?a.map((k)=>k.created_at).sort()[c-1]:null,p={};for(let k of a)for(let R of k.tags||[])p[R]=(p[R]||0)+1;let L=Object.entries(p).sort((k,R)=>R[1]-k[1]).slice(0,5).map(([k,R])=>({tag:k,count:R}));h({ok:!0,total:c,archived:u,with_url:f,with_tags:d,oldest:y,newest:T,top_tags:L,message:`${c} items | ${f} with URL | ${d} with tags`},r.json)});return}let l=nn(t[0]),_=l?` Did you mean '${l}'?`:"";throw F("warn","Unknown command",{input:t[0],suggestion:l}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)un(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);F("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{nn as suggestCommand,cn as sortItems,un as run,en as parseArgs};
|
|
@@ -208,6 +208,21 @@ The command should:
|
|
|
208
208
|
7. Record a run ledger with tool calls, sources, costs, outputs, and generated
|
|
209
209
|
records.
|
|
210
210
|
|
|
211
|
+
## Provider Registry
|
|
212
|
+
|
|
213
|
+
AI provider setup is BYOK and AI SDK v6 based. The local provider layer tracks:
|
|
214
|
+
|
|
215
|
+
- OpenAI via `@ai-sdk/openai`, defaulting to `openai:gpt-5.2`.
|
|
216
|
+
- Anthropic via `@ai-sdk/anthropic`, defaulting to
|
|
217
|
+
`anthropic:claude-sonnet-4-6`.
|
|
218
|
+
- DeepSeek via `@ai-sdk/deepseek`, defaulting to `deepseek:deepseek-chat`.
|
|
219
|
+
|
|
220
|
+
Model aliases live in config and can be inspected with
|
|
221
|
+
`open-knowledge providers models`. Credentials are resolved from env vars by
|
|
222
|
+
default, checked without making provider calls, and usage can be normalized into
|
|
223
|
+
the existing `provider_usage` table for future prompt, embedding, and web-search
|
|
224
|
+
runs.
|
|
225
|
+
|
|
211
226
|
## Non-Goals
|
|
212
227
|
|
|
213
228
|
- Do not make `open-knowledge` own raw source files.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hasna/knowledge",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.11",
|
|
4
4
|
"description": "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"scripts": {
|
|
18
18
|
"test": "bun test",
|
|
19
19
|
"test:cli": "bun test tests/cli.test.ts",
|
|
20
|
-
"build": "bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers src/mcp.js",
|
|
20
|
+
"build": "bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",
|
|
21
21
|
"prepublishOnly": "bun run build",
|
|
22
22
|
"postinstall": "bun run build"
|
|
23
23
|
},
|
|
@@ -50,7 +50,11 @@
|
|
|
50
50
|
"dependencies": {
|
|
51
51
|
"@aws-sdk/client-s3": "^3.1063.0",
|
|
52
52
|
"@aws-sdk/credential-providers": "^3.1063.0",
|
|
53
|
+
"@ai-sdk/anthropic": "^3.0.81",
|
|
54
|
+
"@ai-sdk/deepseek": "^2.0.35",
|
|
55
|
+
"@ai-sdk/openai": "^3.0.68",
|
|
53
56
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
57
|
+
"ai": "^6.0.197",
|
|
54
58
|
"zod": "^4.3.6"
|
|
55
59
|
},
|
|
56
60
|
"devDependencies": {
|