@hasna/knowledge 0.2.11 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/bin/open-knowledge-mcp.js +218 -10
- package/bin/open-knowledge.js +44 -31
- package/docs/architecture/ai-native-knowledge-base.md +12 -0
- package/package.json +1 -1
- package/src/cli.ts +31 -4
- package/src/knowledge-db.ts +2 -0
- package/src/mcp.js +12 -0
- package/src/service.ts +26 -2
- package/src/storage-contract.ts +265 -0
- package/src/wiki-layout.ts +22 -6
package/bin/open-knowledge.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
|
-
var
|
|
4
|
-
`);return t}function
|
|
3
|
+
var D=import.meta.require;import{readFileSync as Q,writeFileSync as J,existsSync as V,renameSync as mt,unlinkSync as Oe}from"fs";import{randomUUID as Ne}from"crypto";import{existsSync as ft,mkdirSync as ue,readFileSync as _t,writeFileSync as pt}from"fs";import{homedir as Se}from"os";import{dirname as gt,join as R,resolve as ht}from"path";var z=R(".hasna","apps","knowledge");function de(){return R(Se(),".open-knowledge","db.json")}function le(){return R(Se(),".hasna","apps","knowledge")}function Et(e=process.cwd()){return ht(e,z)}function B(e){return{home:e,configPath:R(e,"config.json"),jsonStorePath:R(e,"db.json"),knowledgeDbPath:R(e,"knowledge.db"),artifactsDir:R(e,"artifacts"),cacheDir:R(e,"cache"),exportsDir:R(e,"exports"),indexesDir:R(e,"indexes"),logsDir:R(e,"logs"),runsDir:R(e,"runs"),schemasDir:R(e,"schemas"),wikiDir:R(e,"wiki")}}function yt(){return{version:1,mode:"local",storage:{type:"local",artifacts_root:"artifacts"},sources:{preferred_ref:"open-files",allowed_schemes:["open-files","s3","file","https","http"]},providers:{default_model:"openai:gpt-5.2",aliases:{fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"},openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},safety:{network:{web_search_enabled:!1,s3_reads_enabled:!1,allowed_s3_buckets:[]},redaction:{enabled:!0},approvals:{generated_writes_require_approval:!0}}}}function Re(e){let t=B(e);ue(t.home,{recursive:!0});for(let r of[t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir])ue(r,{recursive:!0});if(!ft(t.configPath))pt(t.configPath,`${JSON.stringify(yt(),null,2)}
|
|
4
|
+
`);return t}function xe(e,t=process.cwd()){if(e==="project"||e==="local")return B(Et(t));return B(le())}function G(e){ue(gt(e),{recursive:!0})}function ve(e){let t=_t(e,"utf8");return JSON.parse(t)}function fe(){return B(le()).jsonStorePath}function _e(e){if(!V(e))if(G(e),e===fe()&&V(de()))J(e,Q(de(),"utf8"));else J(e,JSON.stringify({items:[]},null,2))}function Tt(e){return`${e}.lock`}function kt(e,t){let i=Date.now();while(Date.now()-i<5000){try{if(!V(e)){J(e,JSON.stringify({owner:t,ts:Date.now()}));return}let f=JSON.parse(Q(e,"utf8"));if(Date.now()-f.ts>1e4)Oe(e)}catch{}let s=Date.now();while(Date.now()-s<50);}throw Error(`Could not acquire lock on ${e} after 5000ms`)}function bt(e,t){try{if(V(e)){if(JSON.parse(Q(e,"utf8")).owner===t)Oe(e)}}catch{}}function x(e){_e(e);let t=Q(e,"utf8"),r=JSON.parse(t);if(!r||!Array.isArray(r.items))return{items:[]};return r}function U(e,t){let r=`${e}.tmp.${Ne()}`;J(r,JSON.stringify(t,null,2)),mt(r,e)}function v(e,t){let r=Ne(),n=Tt(e);kt(n,r);try{return t()}finally{bt(n,r)}}function pe(){return`k_${Date.now().toString(36)}_${Math.random().toString(36).slice(2,8)}`}function Ae(e){return e.replace(/^k_/,"").slice(0,12)}import{Database as wt}from"bun:sqlite";var St=`
|
|
5
5
|
PRAGMA journal_mode = WAL;
|
|
6
6
|
PRAGMA foreign_keys = ON;
|
|
7
7
|
|
|
@@ -168,7 +168,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
168
168
|
|
|
169
169
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
170
170
|
VALUES (1, datetime('now'));
|
|
171
|
-
`,
|
|
171
|
+
`,Rt=`
|
|
172
172
|
DROP TABLE IF EXISTS chunks_fts;
|
|
173
173
|
|
|
174
174
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
@@ -181,7 +181,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
181
181
|
|
|
182
182
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
183
183
|
VALUES (2, datetime('now'));
|
|
184
|
-
`,
|
|
184
|
+
`,xt=`
|
|
185
185
|
CREATE TABLE IF NOT EXISTS audit_events (
|
|
186
186
|
id TEXT PRIMARY KEY,
|
|
187
187
|
event_type TEXT NOT NULL,
|
|
@@ -212,59 +212,71 @@ CREATE INDEX IF NOT EXISTS idx_approval_gates_status ON approval_gates(status);
|
|
|
212
212
|
|
|
213
213
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
214
214
|
VALUES (3, datetime('now'));
|
|
215
|
-
`;function
|
|
216
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function
|
|
217
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${Ee()}`,t.source_uri??null,t.run_id??null,n.severity,n.type,JSON.stringify({...t.metadata??{},start:n.start,end:n.end}),r]);return t.findings.length}function
|
|
218
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function
|
|
215
|
+
`;function A(e){G(e);let t=new wt(e);return t.exec("PRAGMA foreign_keys = ON;"),t}function C(e){let t=A(e);try{if(t.exec(St),Z(t)<2)t.exec(Rt);if(Z(t)<3)t.exec(xt);return{path:e,schema_version:Z(t)}}finally{t.close()}}function Z(e){return e.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0}function N(e,t){return e.query(`SELECT COUNT(*) AS n FROM ${t}`).get()?.n??0}function Le(e){let t=A(e);try{return{schema_version:Z(t),sources:N(t,"sources"),source_revisions:N(t,"source_revisions"),chunks:N(t,"chunks"),wiki_pages:N(t,"wiki_pages"),citations:N(t,"citations"),indexes:N(t,"knowledge_indexes"),runs:N(t,"runs"),run_events:N(t,"run_events"),redaction_findings:N(t,"redaction_findings"),audit_events:N(t,"audit_events"),approval_gates:N(t,"approval_gates"),storage_objects:N(t,"storage_objects")}}finally{t.close()}}import{existsSync as vt,mkdirSync as Ie,readFileSync as Ot,writeFileSync as Nt}from"fs";import{dirname as At,join as ge,relative as Lt,sep as It}from"path";function H(e){let t=e.replace(/\\/g,"/").trim();if(!t||t.startsWith("/"))throw Error(`Invalid artifact key: ${e}`);let r=t.split("/").filter(Boolean);if(r.length===0||r.some((n)=>n==="."||n===".."))throw Error(`Invalid artifact key: ${e}`);return r.join("/")}function he(e,t){let r=Lt(e,t);if(r.startsWith("..")||r===".."||r.startsWith(`..${It}`))throw Error(`Artifact path escapes root: ${t}`)}class De{root;type="local";canRead=!0;canWrite=!0;constructor(e){this.root=e;Ie(e,{recursive:!0})}async put(e){let t=H(e.key),r=ge(this.root,t);return he(this.root,r),Ie(At(r),{recursive:!0}),Nt(r,e.body),{key:t,uri:`file://${r}`}}async getText(e){let t=H(e),r=ge(this.root,t);return he(this.root,r),Ot(r,"utf8")}async exists(e){let t=H(e),r=ge(this.root,t);return he(this.root,r),vt(r)}}class Ce{options;type="s3";canRead=!0;canWrite=!0;client;constructor(e){this.options=e;this.client=e.client}async getClient(){if(this.client)return this.client;let[{S3Client:e},{fromIni:t}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]);return this.client=new e({region:this.options.region,credentials:this.options.profile?t({profile:this.options.profile}):void 0,maxAttempts:this.options.max_attempts}),this.client}objectKey(e){let t=H(e),r=this.options.prefix?H(this.options.prefix):"";return r?`${r}/${t}`:t}async put(e){let[{PutObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e.key);return await r.send(new t({Bucket:this.options.bucket,Key:n,Body:e.body,ContentType:e.content_type,Metadata:e.metadata,ServerSideEncryption:this.options.server_side_encryption,SSEKMSKeyId:this.options.kms_key_id})),{key:n,uri:`s3://${this.options.bucket}/${n}`}}async getText(e){let[{GetObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e),i=await r.send(new t({Bucket:this.options.bucket,Key:n}));if(!i.Body)return"";return await i.Body.transformToString()}async exists(e){let[{HeadObjectCommand:t},r]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),n=this.objectKey(e);try{return await r.send(new t({Bucket:this.options.bucket,Key:n})),!0}catch(i){let s=i instanceof Error?i.name:"";if(s==="NotFound"||s==="NoSuchKey"||s==="NotFoundError")return!1;throw i}}}function Ue(e,t){if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)throw Error("S3 artifact storage requires storage.s3.bucket");return new Ce({bucket:e.storage.s3.bucket,prefix:e.storage.s3.prefix,region:e.storage.s3.region,profile:e.storage.s3.profile,max_attempts:e.storage.s3.max_attempts,server_side_encryption:e.storage.s3.server_side_encryption,kms_key_id:e.storage.s3.kms_key_id})}return new De(t.artifactsDir)}import{createHash as Bt,randomUUID as zt}from"crypto";import{existsSync as Ht,readFileSync as qt}from"fs";import{basename as Yt}from"path";function Pe(e,t){if(!e)throw Error(t);return e}function Dt(e){let r=e.slice(13).split("/").filter(Boolean),n=r[0];if(n!=="file"&&n!=="source")throw Error("Invalid open-files ref. Expected open-files://file/<id>, open-files://file/<id>/revision/<revision_id>, or open-files://source/<id>/path/<path>.");let i=Pe(r[1],"Invalid open-files ref. Missing id.");if(n==="file"){if(r.length===2)return{kind:"open-files",uri:e,entity:n,id:i};if(r[2]==="revision"&&r[3]&&r.length===4)return{kind:"open-files",uri:e,entity:n,id:i,revision_id:decodeURIComponent(r[3])};throw Error("Invalid open-files file ref. Expected open-files://file/<id>/revision/<revision_id>.")}let s=r.indexOf("path"),f=s>=0?decodeURIComponent(r.slice(s+1).join("/")):void 0;return{kind:"open-files",uri:e,entity:n,id:i,path:f}}function Ct(e){let t=new URL(e),r=Pe(t.hostname,"Invalid s3 ref. Missing bucket."),n=decodeURIComponent(t.pathname.replace(/^\/+/,""));if(!n)throw Error("Invalid s3 ref. Missing object key.");return{kind:"s3",uri:e,bucket:r,key:n}}function Ut(e){let t=new URL(e);return{kind:"file",uri:e,path:decodeURIComponent(t.pathname)}}function Pt(e){let t=new URL(e);return{kind:"web",uri:e,url:t.toString()}}function L(e){if(e.startsWith("open-files://"))return Dt(e);if(e.startsWith("s3://"))return Ct(e);if(e.startsWith("file://"))return Ut(e);if(e.startsWith("https://")||e.startsWith("http://"))return Pt(e);throw Error(`Unsupported source ref scheme: ${e}`)}function je(e,t=L(e)){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Ke(e){let t=L(e);return t.kind==="open-files"&&t.entity==="file"?t.revision_id??null:null}import{createHash as jt,randomUUID as Ee}from"crypto";import{relative as Kt,resolve as Xe,sep as Ft}from"path";function Fe(e){let t=process.env[e];return t==="1"||t==="true"||t==="yes"}function Me(e,t){let r=e,n=new Set(r.safety?.network?.allowed_s3_buckets??[]);if(e.storage.type==="s3"&&e.storage.s3?.bucket)n.add(e.storage.s3.bucket);if(process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS)for(let i of process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.split(",").map((s)=>s.trim()).filter(Boolean))n.add(i);return{mode:e.mode,allowWriteRoots:[t.home,t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir].map((i)=>Xe(i)),readOnlySourceAccess:!0,network:{webSearchEnabled:r.safety?.network?.web_search_enabled??Fe("HASNA_KNOWLEDGE_WEB_SEARCH"),s3ReadsEnabled:r.safety?.network?.s3_reads_enabled??Fe("HASNA_KNOWLEDGE_ALLOW_S3_READS"),allowedS3Buckets:[...n].sort()},redaction:{enabled:r.safety?.redaction?.enabled??!0},approvals:{generatedWritesRequireApproval:r.safety?.approvals?.generated_writes_require_approval??!0}}}function Xt(e,t){let r=Kt(e,t);return r===""||!r.startsWith("..")&&r!==".."&&!r.startsWith(`..${Ft}`)}function K(e,t){let r=Xe(e);if(!t.allowWriteRoots.some((n)=>Xt(n,r)))throw Error(`Safety policy denied write outside .hasna/apps/knowledge: ${e}`)}function j(e,t){let n=new URL(e).hostname;if(!t.network.s3ReadsEnabled)throw Error("Safety policy denied S3 read. Set safety.network.s3_reads_enabled=true or HASNA_KNOWLEDGE_ALLOW_S3_READS=1.");if(!t.network.allowedS3Buckets.includes(n))throw Error(`Safety policy denied S3 bucket "${n}". Add it to safety.network.allowed_s3_buckets or HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.`)}function ee(e){if(!e.network.webSearchEnabled)throw Error("Safety policy denied web search. Set safety.network.web_search_enabled=true or HASNA_KNOWLEDGE_WEB_SEARCH=1.")}var Mt=[{type:"private_key_block",severity:"high",regex:/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,replacement:"[REDACTED:private_key_block]"},{type:"secret_assignment",severity:"high",regex:/\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*['"]?[^'"\s]{8,}/gi,replacement:"[REDACTED:secret_assignment]"},{type:"openai_api_key",severity:"high",regex:/\bsk-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:openai_api_key]"},{type:"anthropic_api_key",severity:"high",regex:/\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:anthropic_api_key]"},{type:"aws_access_key_id",severity:"high",regex:/\bA(?:KIA|SIA)[A-Z0-9]{16}\b/g,replacement:"[REDACTED:aws_access_key_id]"}];function te(e,t){if(t&&!t.redaction.enabled)return{text:e,findings:[]};let r=e,n=[];for(let i of Mt)r=r.replace(i.regex,(s,...f)=>{let _=typeof f.at(-2)==="number"?f.at(-2):r.indexOf(s);return n.push({type:i.type,severity:i.severity,start:Math.max(0,_),end:Math.max(0,_+s.length)}),i.replacement});return{text:r,findings:n}}function Wt(e){return`audit_${jt("sha256").update(`${e.event_type}\x00${e.action}\x00${e.target_uri??""}\x00${e.created_at??""}\x00${JSON.stringify(e.metadata??{})}\x00${Ee()}`).digest("hex").slice(0,24)}`}function w(e,t){let r=t.created_at??new Date().toISOString(),n=Wt({...t,created_at:r});return e.run(`INSERT INTO audit_events (id, event_type, action, target_uri, decision, metadata_json, created_at)
|
|
216
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[n,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),r]),n}function re(e,t){let r=t.created_at??new Date().toISOString();for(let n of t.findings)e.run(`INSERT INTO redaction_findings (id, source_uri, run_id, severity, finding_type, metadata_json, created_at)
|
|
217
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${Ee()}`,t.source_uri??null,t.run_id??null,n.severity,n.type,JSON.stringify({...t.metadata??{},start:n.start,end:n.end}),r]);return t.findings.length}function We(e,t){let r=t.created_at??new Date().toISOString(),n=`approval_${Ee()}`;return e.run(`INSERT INTO approval_gates (id, action, target_uri, status, reason, approved_by, metadata_json, created_at, updated_at)
|
|
218
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),r,r]),{id:n,status:"approved"}}function $t(e,t,r){let n=e.query(`SELECT id FROM approval_gates
|
|
219
219
|
WHERE action = ? AND status = 'approved' AND (target_uri IS NULL OR target_uri = ? OR ? IS NULL)
|
|
220
|
-
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function
|
|
220
|
+
ORDER BY updated_at DESC LIMIT 1`).get(t,r??null,r??null);return Boolean(n)}function $e(e,t,r,n){let i=r==="generated_write"&&t.approvals.generatedWritesRequireApproval,s=!i||$t(e,r,n);return{action:r,target_uri:n??null,approval_required:i,approved:s,decision:s?"allow":"requires_approval"}}function ne(e,t){return`${e}_${Bt("sha256").update(t).digest("hex").slice(0,20)}`}function X(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function b(e){return typeof e==="string"&&e.length>0?e:void 0}function Gt(e){let t=b(e.source_ref)??b(e.source_uri)??b(e.uri);if(t)return t;let r=b(e.file_id);if(r){let s=b(e.revision_id)??b(e.revision),f=`open-files://file/${encodeURIComponent(r)}`;return s?`${f}/revision/${encodeURIComponent(s)}`:f}let n=b(e.source_id),i=b(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Outbox event is missing source_ref, file_id, or source_id/path.")}function Jt(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function Vt(e){return b(e.hash)??b(e.checksum)??b(e.sha256)??null}function Qt(e,t,r){return b(e.revision_id)??b(e.revision)??b(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??null}function Zt(e){return(b(e.event)??b(e.type)??b(e.action)??b(e.change_type)??"changed").toLowerCase()}function er(e){let t=b(e.path);return b(e.title)??b(e.name)??(t?Yt(t):null)}function tr(e,t){let r=Gt(e),n=L(r),i=Vt(e);return{raw:e,eventType:Zt(e),sourceRef:r,sourceUri:Jt(r,n),kind:n.kind,title:er(e),revision:Qt(e,n,i),hash:i,status:b(e.status)?.toLowerCase()??null,updatedAt:b(e.updated_at)??t,acl:e.permissions??e.acl??void 0}}function rr(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Outbox array parse failed.");return r.map((n)=>{let i=X(n);if(!i)throw Error("Outbox array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=X(r);if(!n)throw Error("Outbox object parse failed.");if(Array.isArray(n.events))return n.events.map((i)=>{let s=X(i);if(!s)throw Error("Outbox events entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=X(JSON.parse(i));if(!s)throw Error("Outbox JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=X(JSON.parse(r));if(!n)throw Error("Outbox JSONL entries must be objects.");return n})}async function nr(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 outbox URI: ${e}`);if(r)j(e,r);let[{S3Client:f,GetObjectCommand:_},{fromIni:a}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),o=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new f({region:o?.region,credentials:o?.profile?a({profile:o.profile}):void 0,maxAttempts:o?.max_attempts}).send(new _({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function ir(e,t,r){if(e.startsWith("s3://"))return nr(e,t,r);if(!Ht(e))throw Error(`Outbox not found: ${e}`);return qt(e,"utf8")}function Be(e,t){let r={};if(e)try{r=X(JSON.parse(e))??{}}catch{r={}}return JSON.stringify({...r,...t})}function sr(e,t,r){let n=ne("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
221
221
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
222
222
|
ON CONFLICT(uri) DO UPDATE SET
|
|
223
223
|
kind = excluded.kind,
|
|
224
224
|
title = COALESCE(excluded.title, sources.title),
|
|
225
|
-
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(b(t.raw.path))s.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[
|
|
225
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(b(t.raw.path))s.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[Be(i.metadata_json,s),t.acl===void 0?null:JSON.stringify(t.acl),t.acl===void 0?null:JSON.stringify(t.acl),t.updatedAt,i.id]),i.id}function or(e,t,r,n){if(!r.revision)return null;let i=ne("rev",`${t}\x00${r.revision}`),s={source_ref:r.sourceRef,source_uri:r.sourceUri,status:r.status,last_outbox_event:r.eventType,reindex_required:!0};return e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
226
226
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
227
227
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
228
228
|
hash = COALESCE(excluded.hash, source_revisions.hash),
|
|
229
|
-
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,b(r.raw.extracted_text_ref)??null,JSON.stringify(s),n]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision)?.id??null}function
|
|
230
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:n.length}),t,t]);let
|
|
231
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[
|
|
232
|
-
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[
|
|
233
|
-
`);if(!n.trim())return[];let i=[],s=0;while(s<n.length){let
|
|
229
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,b(r.raw.extracted_text_ref)??null,JSON.stringify(s),n]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision)?.id??null}function ar(e,t,r){if(r.revision)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").all(t,r.revision).map((n)=>n.id);if(r.hash)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND hash = ?").all(t,r.hash).map((n)=>n.id);return e.query("SELECT id FROM source_revisions WHERE source_id = ?").all(t).map((n)=>n.id)}function cr(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t),n=0;for(let s of r){let f=e.query("SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?").get(s.id);n+=f?.n??0,e.run("DELETE FROM chunk_embeddings WHERE chunk_id = ?",[s.id]),e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[s.id])}e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]);let i=e.query("SELECT metadata_json FROM source_revisions WHERE id = ?").get(t);return e.run("UPDATE source_revisions SET metadata_json = ? WHERE id = ?",[Be(i?.metadata_json,{reindex_required:!0,invalidated_at:new Date().toISOString()}),t]),{chunksDeleted:r.length,embeddingsDeleted:n}}function ur(e,t){return t==="deleted"||["delete","deleted","remove","removed"].includes(e)}function dr(e){return["move","moved","rename","renamed","path_changed"].includes(e)}function lr(e){return["permission","permissions","permission_changed","acl_changed"].includes(e)}async function ze(e){let t=(e.now??new Date).toISOString();if(e.safetyPolicy)K(e.dbPath,e.safetyPolicy);C(e.dbPath);let r=await ir(e.input,e.config,e.safetyPolicy),n=rr(r),i=A(e.dbPath),s=`run_${zt()}`;try{return i.transaction(()=>{i.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
230
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:n.length}),t,t]);let f=new Set,_=new Set,a=0,o=0,c=0,u=0,l=0,d=0;return w(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:n.length,read_only:!0},created_at:t}),n.forEach((y,m)=>{let p=tr(y,t),I=sr(i,p,t);f.add(I);let k=or(i,I,p,t);if(k)_.add(k);let S=ar(i,I,p);for(let g of S){_.add(g);let O=cr(i,g);a+=O.chunksDeleted,o+=O.embeddingsDeleted,c+=1}if(ur(p.eventType,p.status))u+=1;if(dr(p.eventType))l+=1;if(lr(p.eventType)||p.acl!==void 0)d+=1;i.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
231
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[ne("evt",`${s}\x00${m}\x00${p.sourceRef}\x00${p.eventType}`),s,"info",p.eventType,JSON.stringify({source_ref:p.sourceRef,source_uri:p.sourceUri,revision:p.revision,hash:p.hash,status:p.status,affected_revisions:S.length}),p.updatedAt])}),i.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
232
|
+
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[ne("usage",s),s,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),w(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:s,events:n.length,sources:f.size,revisions:_.size,chunks_deleted:a,embeddings_deleted:o},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:s,events_seen:n.length,sources_touched:f.size,revisions_touched:_.size,chunks_deleted:a,embeddings_deleted:o,stale_revisions:c,deleted_sources:u,moved_sources:l,permission_updates:d}})()}finally{i.close()}}import{createHash as fr}from"crypto";import{existsSync as _r,readFileSync as pr}from"fs";import{basename as gr}from"path";function ye(e,t){return`${e}_${fr("sha256").update(t).digest("hex").slice(0,20)}`}function M(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function h(e){return typeof e==="string"&&e.length>0?e:void 0}function hr(e){return typeof e==="number"&&Number.isFinite(e)?e:void 0}function Er(e){let t=h(e.source_ref)??h(e.source_uri)??h(e.uri);if(t)return t;let r=h(e.file_id);if(r){let s=h(e.revision_id)??h(e.revision),f=`open-files://file/${encodeURIComponent(r)}`;return s?`${f}/revision/${encodeURIComponent(s)}`:f}let n=h(e.source_id),i=h(e.path);if(n&&i)return`open-files://source/${encodeURIComponent(n)}/path/${encodeURIComponent(i)}`;throw Error("Manifest item is missing source_ref, file_id, or source_id/path.")}function yr(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function mr(e){let t=h(e.extracted_text)??h(e.text)??h(e.content_text)??h(e.markdown);if(t!==void 0)return t;let r=e.content;return typeof r==="string"?r:null}function Tr(e){let t=h(e.extracted_text_ref)??h(e.extracted_text_uri)??h(e.text_ref);if(t)return t;let r=M(e.content);return h(r?.extracted_text_ref)??h(r?.extracted_text_uri)??null}function kr(e){let t=h(e.path);return h(e.title)??h(e.name)??(t?gr(t):null)}function br(e){return h(e.hash)??h(e.checksum)??h(e.sha256)??null}function wr(e,t,r){return h(e.revision_id)??h(e.revision)??h(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??r??h(e.updated_at)??"current"}function Sr(e,t){let r={};for(let[n,i]of Object.entries(e)){if(["text","content","content_text","extracted_text","markdown"].includes(n))continue;r[n]=i}return r.source_ref=t.sourceRef,r.source_uri=t.sourceUri,r.status=t.status,r}function Rr(e,t){let r=Er(e),n=L(r),i=yr(r,n),s=br(e),f=h(e.status)??"active";return{raw:e,sourceRef:r,sourceUri:i,kind:n.kind,title:kr(e),revision:wr(e,n,s),hash:s,extractedTextUri:Tr(e),text:mr(e),metadata:Sr(e,{sourceRef:r,sourceUri:i,status:f}),acl:e.permissions??e.acl??{},status:f,updatedAt:h(e.updated_at)??t}}function xr(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let r=JSON.parse(t);if(!Array.isArray(r))throw Error("Manifest array parse failed.");return r.map((n)=>{let i=M(n);if(!i)throw Error("Manifest array entries must be objects.");return i})}if(t.startsWith("{"))try{let r=JSON.parse(t),n=M(r);if(!n)throw Error("Manifest object parse failed.");if(Array.isArray(n.items))return n.items.map((i)=>{let s=M(i);if(!s)throw Error("Manifest items entries must be objects.");return s});if("source_ref"in n||"source_uri"in n||"file_id"in n)return[n]}catch(r){let n=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(n.length<=1)throw r;return n.map((i)=>{let s=M(JSON.parse(i));if(!s)throw Error("Manifest JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((r)=>r.trim().length>0).map((r)=>{let n=M(JSON.parse(r));if(!n)throw Error("Manifest JSONL entries must be objects.");return n})}async function vr(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 manifest URI: ${e}`);if(r)j(e,r);let[{S3Client:f,GetObjectCommand:_},{fromIni:a}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),o=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new f({region:o?.region,credentials:o?.profile?a({profile:o.profile}):void 0,maxAttempts:o?.max_attempts}).send(new _({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function Or(e,t,r){if(e.startsWith("s3://"))return vr(e,t,r);if(!_r(e))throw Error(`Manifest not found: ${e}`);return pr(e,"utf8")}function Nr(e,t,r){let n=e.replace(/\r\n/g,`
|
|
233
|
+
`);if(!n.trim())return[];let i=[],s=0;while(s<n.length){let f=Math.min(n.length,s+t),_=f;if(f<n.length){let o=n.lastIndexOf(`
|
|
234
234
|
|
|
235
|
-
`,
|
|
235
|
+
`,f),c=n.lastIndexOf(". ",f),u=Math.max(o,c);if(u>s+Math.floor(t*0.5))_=u+(u===o?2:1)}let a=n.slice(s,_).trim();if(a)i.push({ordinal:i.length,text:a,startOffset:s,endOffset:_});if(_>=n.length)break;s=Math.max(0,_-r)}return i}function Ar(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Lr(e,t){let r=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t);for(let n of r)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[n.id]);return e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]),r.length}function Ir(e,t,r){let n=ye("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
236
236
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
237
237
|
ON CONFLICT(uri) DO UPDATE SET
|
|
238
238
|
kind = excluded.kind,
|
|
239
239
|
title = excluded.title,
|
|
240
240
|
metadata_json = excluded.metadata_json,
|
|
241
241
|
acl_json = excluded.acl_json,
|
|
242
|
-
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function
|
|
242
|
+
updated_at = excluded.updated_at`,[n,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),r,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function Dr(e,t,r,n){let i=ye("rev",`${t}\x00${r.revision}`);e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
243
243
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
244
244
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
245
245
|
hash = excluded.hash,
|
|
246
246
|
extracted_text_uri = excluded.extracted_text_uri,
|
|
247
|
-
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision);if(!s)throw Error(`Failed to upsert source revision: ${r.sourceRef}`);return s.id}function
|
|
248
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[c,t,"source",
|
|
247
|
+
metadata_json = excluded.metadata_json`,[i,t,r.revision,r.hash,r.extractedTextUri,JSON.stringify(r.metadata),n]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,r.revision);if(!s)throw Error(`Failed to upsert source revision: ${r.sourceRef}`);return s.id}function Cr(e,t,r,n,i,s,f){if(!r.text||r.status.toLowerCase()==="deleted")return{chunksInserted:0,redactions:0};let _=te(r.text,f);if(_.findings.length>0)re(e,{source_uri:r.sourceUri,findings:_.findings,metadata:{source_ref:r.sourceRef,revision:r.revision},created_at:n}),w(e,{event_type:"redaction",action:"source_text_redact",target_uri:r.sourceUri,decision:"redacted",metadata:{findings:_.findings.length,source_ref:r.sourceRef,revision:r.revision},created_at:n});let a=Nr(_.text,i,s);for(let o of a){let c=ye("chk",`${t}\x00${o.ordinal}\x00${o.text}`),u={source_ref:r.sourceRef,source_uri:r.sourceUri,hash:r.hash,status:r.status,path:h(r.raw.path)??null,mime:h(r.raw.mime)??h(r.raw.content_type)??null,size:hr(r.raw.size)??null};e.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
248
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[c,t,"source",o.ordinal,o.text,Ar(o.text),o.startOffset,o.endOffset,JSON.stringify(u),n]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[c,o.text,r.title??"",r.sourceUri])}return{chunksInserted:a.length,redactions:_.findings.length}}async function He(e){let t=e.now??new Date;if(e.safetyPolicy)K(e.dbPath,e.safetyPolicy);C(e.dbPath);let r=await Or(e.input,e.config,e.safetyPolicy),n=xr(r);return me({dbPath:e.dbPath,items:n,sourceLabel:e.input,safetyPolicy:e.safetyPolicy,now:t,maxChunkChars:e.maxChunkChars,chunkOverlapChars:e.chunkOverlapChars})}async function me(e){let t=(e.now??new Date).toISOString(),r=e.maxChunkChars??4000,n=e.chunkOverlapChars??200;if(r<500)throw Error("maxChunkChars must be at least 500.");if(n<0||n>=r)throw Error("chunkOverlapChars must be less than maxChunkChars.");if(e.safetyPolicy)K(e.dbPath,e.safetyPolicy);C(e.dbPath);let i=A(e.dbPath);try{return i.transaction(()=>{let f=new Set,_=new Set,a=0,o=0,c=0,u=0;w(i,{event_type:"source_read",action:e.readAction??(e.sourceLabel.startsWith("s3://")?"s3_manifest_read":"local_manifest_read"),target_uri:e.sourceLabel,decision:"allow",metadata:{items:e.items.length,read_only:!0},created_at:t});for(let l of e.items){let d=Rr(l,t),y=Ir(i,d,t),m=Dr(i,y,d,t);if(f.add(y),_.add(m),d.text||d.status.toLowerCase()==="deleted")o+=Lr(i,m);let p=Cr(i,m,d,t,r,n,e.safetyPolicy);a+=p.chunksInserted,c+=p.redactions}return w(i,{event_type:"write",action:"knowledge_manifest_ingest",target_uri:e.dbPath,decision:"allow",metadata:{items:e.items.length,sources:f.size,revisions:_.size,chunks_inserted:a,redactions:c},created_at:t}),{path:e.sourceLabel,db_path:e.dbPath,items_seen:e.items.length,sources_upserted:f.size,revisions_upserted:_.size,chunks_inserted:a,chunks_deleted:o,redactions:c,skipped:u}})()}finally{i.close()}}import{createHash as Mr}from"crypto";import{existsSync as Wr,readFileSync as $r}from"fs";import{basename as oe}from"path";function ie(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function W(e,t){for(let r of t){let n=e[r];if(typeof n==="string"&&n.length>0)return n}return null}function qe(e,t){for(let r of t){let n=e[r];if(typeof n==="number"&&Number.isFinite(n))return n}return null}function Ur(e,t){let r=e.mode;if(typeof r==="string"&&r!=="read_only")throw Error(`Source resolver denied ${t}. Permission mode is ${r}, expected read_only.`);let n=e.denied_purposes;if(Array.isArray(n)&&n.includes(t))throw Error(`Source resolver denied ${t}. Purpose is explicitly denied.`);let i=e.allowed_purposes;if(Array.isArray(i)&&i.length>0&&!i.includes(t))throw Error(`Source resolver denied ${t}. Allowed purposes: ${i.join(", ")}`)}function Pr(e,t,r){if(!t)return r;try{let n=L(e);if(n.kind==="open-files"&&n.entity==="file")return`${e}/revision/${encodeURIComponent(t.revision)}`}catch{return r}return r}function jr(e,t,r){return e.query(`SELECT id, uri, kind, title, metadata_json, acl_json, updated_at
|
|
249
249
|
FROM sources
|
|
250
250
|
WHERE uri = ? OR uri = ?
|
|
251
251
|
ORDER BY CASE WHEN uri = ? THEN 0 ELSE 1 END
|
|
252
|
-
LIMIT 1`).get(t,r,t)??null}function
|
|
252
|
+
LIMIT 1`).get(t,r,t)??null}function Kr(e,t,r){if(r)return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
253
253
|
FROM source_revisions
|
|
254
254
|
WHERE source_id = ? AND revision = ?
|
|
255
255
|
LIMIT 1`).get(t,r)??null;return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
256
256
|
FROM source_revisions
|
|
257
257
|
WHERE source_id = ?
|
|
258
258
|
ORDER BY created_at DESC, revision DESC
|
|
259
|
-
LIMIT 1`).get(t)??null}function
|
|
259
|
+
LIMIT 1`).get(t)??null}function Fr(e,t){if(!t)return 0;return e.query("SELECT COUNT(*) AS n FROM chunks WHERE source_revision_id = ?").get(t)?.n??0}function Xr(e,t,r){if(!t||r<=0)return[];return e.query(`SELECT id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json
|
|
260
260
|
FROM chunks
|
|
261
261
|
WHERE source_revision_id = ?
|
|
262
262
|
ORDER BY ordinal ASC
|
|
263
|
-
LIMIT ?`).all(t,r)}async function
|
|
263
|
+
LIMIT ?`).all(t,r)}async function se(e){let t=e.purpose??"knowledge_answer",r=Math.max(0,Math.min(e.limit??10,100)),n=(e.now??new Date).toISOString(),i=L(e.sourceRef),s=je(e.sourceRef,i),f=Ke(e.sourceRef);if(e.safetyPolicy){if(!e.safetyPolicy.readOnlySourceAccess)throw Error("Safety policy denied source resolution.");K(e.dbPath,e.safetyPolicy)}C(e.dbPath);let _=A(e.dbPath);try{return _.transaction(()=>{let a=jr(_,s,e.sourceRef);if(!a)return w(_,{event_type:"source_read",action:"open_files_resolve_missing",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:s},created_at:n}),{source_ref:e.sourceRef,source_uri:s,purpose:t,read_only:!0,resolved:!1,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:null,revision:null,content:{mime:null,size:null,hash:null,text_available:!1,chunks_total:0,chunks_returned:0,char_count_returned:0,extracted_text_ref:null,bytes_available:!1,bytes_exposed:!1},chunks:[],citations:[]};let o=ie(a.metadata_json),c=ie(a.acl_json);try{Ur(c,t)}catch(g){throw w(_,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"deny",metadata:{purpose:t,read_only:!0,source_uri:a.uri,error:g instanceof Error?g.message:String(g)},created_at:n}),g}let u=Kr(_,a.id,f),l=ie(u?.metadata_json),d=Fr(_,u?.id??null),y=Xr(_,u?.id??null,r),m=Pr(a.uri,u,e.sourceRef),p=y.map((g)=>{let O=ie(g.metadata_json),T={resolver:"open-files-read-only",mode:"local_catalog",purpose:t,read_only:!0,source_ref:W(O,["source_ref"])??m,source_uri:a.uri,source_revision_id:u?.id??null,revision:u?.revision??null,hash:u?.hash??W(O,["hash"]),chunk_id:g.id,start_offset:g.start_offset,end_offset:g.end_offset,resolved_at:n};return{id:g.id,kind:g.kind,ordinal:g.ordinal,text:g.text,token_count:g.token_count,start_offset:g.start_offset,end_offset:g.end_offset,metadata:O,evidence:T}}),I=p.map((g)=>({source_ref:g.evidence.source_ref,source_uri:a.uri,chunk_id:g.id,quote:g.text.slice(0,500),start_offset:g.start_offset,end_offset:g.end_offset,evidence:g.evidence}));w(_,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:a.uri,revision:u?.revision??null,chunks_returned:p.length,chunks_total:d},created_at:n});let k=W(o,["mime","content_type"])??W(l,["mime","content_type"]),S=qe(o,["size","size_bytes"])??qe(l,["size","size_bytes"]);return{source_ref:m,source_uri:a.uri,purpose:t,read_only:!0,resolved:!0,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:{id:a.id,uri:a.uri,kind:a.kind,title:a.title,metadata:o,permissions:c,updated_at:a.updated_at},revision:u?{id:u.id,revision:u.revision,hash:u.hash,extracted_text_uri:u.extracted_text_uri,metadata:l,created_at:u.created_at,reindex_required:l.reindex_required===!0}:null,content:{mime:k,size:S,hash:u?.hash??W(o,["hash","checksum","sha256"]),text_available:d>0,chunks_total:d,chunks_returned:p.length,char_count_returned:p.reduce((g,O)=>g+O.text.length,0),extracted_text_ref:u?.extracted_text_uri??W(l,["extracted_text_ref","extracted_text_uri"]),bytes_available:!1,bytes_exposed:!1},chunks:p,citations:I}})()}finally{_.close()}}function $(e){return`sha256:${Mr("sha256").update(e).digest("hex")}`}function Br(e){return e.replace(/<script[\s\S]*?<\/script>/gi," ").replace(/<style[\s\S]*?<\/style>/gi," ").replace(/<[^>]+>/g," ").replace(/ /g," ").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/\s+\n/g,`
|
|
264
264
|
`).replace(/\n\s+/g,`
|
|
265
|
-
`).replace(/[ \t]{2,}/g," ").trim()}async function
|
|
266
|
-
|
|
267
|
-
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??$(r),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function
|
|
265
|
+
`).replace(/[ \t]{2,}/g," ").trim()}async function zr(e,t,r){let n=new URL(e),i=n.hostname,s=decodeURIComponent(n.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 source URI: ${e}`);if(r)j(e,r);let[{S3Client:f,GetObjectCommand:_},{fromIni:a}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),o=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,u=await new f({region:o?.region,credentials:o?.profile?a({profile:o.profile}):void 0,maxAttempts:o?.max_attempts}).send(new _({Bucket:i,Key:s}));if(!u.Body)return"";return await u.Body.transformToString()}async function Hr(e,t){if(t)ee(t);let r=await fetch(e,{headers:{accept:"text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5","user-agent":"@hasna/knowledge source-ingest"}});if(!r.ok)throw Error(`Web source read failed ${r.status}: ${e}`);let n=r.headers.get("content-type"),i=await r.text();return{text:n?.includes("html")?Br(i):i,mime:n}}function ae(e){if(e.kind==="file")return oe(e.path);if(e.kind==="s3")return oe(e.key);if(e.kind==="web")return oe(new URL(e.url).pathname)||e.url;return e.path?oe(e.path):e.id}async function Ye(e,t,r){if(e.kind==="file"){if(!Wr(e.path))throw Error(`Source file not found: ${e.path}`);let n=$r(e.path,"utf8");return{text:n,contentSource:"file",title:ae(e),mime:"text/plain",size:n.length,hash:$(n),revision:null,extractedTextRef:null,metadata:{path:e.path},permissions:{mode:"read_only"}}}if(e.kind==="s3"){let n=await zr(e.uri,t,r);return{text:n,contentSource:"s3",title:ae(e),mime:"text/plain",size:n.length,hash:$(n),revision:null,extractedTextRef:null,metadata:{bucket:e.bucket,key:e.key},permissions:{mode:"read_only"}}}if(e.kind==="web"){let n=await Hr(e.url,r);return{text:n.text,contentSource:"web",title:ae(e),mime:n.mime,size:n.text.length,hash:$(n.text),revision:null,extractedTextRef:null,metadata:{url:e.url},permissions:{mode:"read_only"}}}throw Error(`Direct source reading is not available for ${e.uri}`)}async function qr(e,t,r){if(e.startsWith("open-files://"))throw Error("Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.");let n=L(e);return{text:(await Ye(n,t,r)).text,contentSource:"extracted_text_ref"}}async function Yr(e){let t=await se({dbPath:e.dbPath,sourceRef:e.sourceRef,purpose:e.purpose??"knowledge_index",limit:100,safetyPolicy:e.safetyPolicy,now:e.now});if(!t.resolved)throw Error("Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.");if(t.revision?.extracted_text_uri&&!t.content.text_available){let n=await qr(t.revision.extracted_text_uri,e.config,e.safetyPolicy);return{text:n.text,contentSource:n.contentSource,title:t.source?.title??null,mime:t.content.mime,size:n.text.length,hash:t.revision.hash??$(n.text),revision:t.revision.revision,extractedTextRef:t.revision.extracted_text_uri,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}if(t.chunks.length===0)throw Error("Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.");let r=t.chunks.map((n)=>n.text).join(`
|
|
266
|
+
|
|
267
|
+
`);return{text:r,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:r.length,hash:t.revision?.hash??$(r),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function Gr(e,t,r,n){let i=r.hash??$(r.text),s={...r.metadata,source_ref:e,content_source:r.contentSource,read_only:!0},f={source_ref:e,name:r.title??ae(t),mime:r.mime??"text/plain",size:r.size??r.text.length,hash:i,revision:r.revision??i,status:"active",updated_at:new Date().toISOString(),permissions:{mode:"read_only",allowed_purposes:[n],...r.permissions},metadata:s,extracted_text_ref:r.extractedTextRef,extracted_text:r.text};if(t.kind==="open-files"){if(t.entity==="file")f.file_id=t.id;if(t.entity==="source")f.source_id=t.id,f.path=t.path}if(t.kind==="file")f.path=t.path;if(t.kind==="s3")f.path=t.key;if(t.kind==="web")f.url=t.url;return f}async function Ge(e){let t=e.purpose??"knowledge_index",r=L(e.sourceRef),n=r.kind==="open-files"?await Yr(e):await Ye(r,e.config,e.safetyPolicy),i=Gr(e.sourceRef,r,n,t);return{...await me({dbPath:e.dbPath,items:[i],sourceLabel:e.sourceRef,readAction:"source_ref_ingest_read",safetyPolicy:e.safetyPolicy,now:e.now}),source_ref:e.sourceRef,content_source:n.contentSource,read_only:!0,hash:String(i.hash)}}var Je={openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},Jr={openai:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!0,reasoning:!0,embeddings:!0},anthropic:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!1,reasoning:!0,embeddings:!1},deepseek:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!1,native_web_search:!1,reasoning:!0,embeddings:!1}},Vr={default:"openai:gpt-5.2",fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"};function Ve(e){return e.providers??{}}function Qr(e,t){let r=Ve(e)[t]??{};return{...Je[t],...r}}function Qe(e){let t=Ve(e);return{...Vr,...t.default_model?{default:t.default_model}:{},...t.aliases??{}}}function Te(e){let[t,...r]=e.split(":"),n=r.join(":");if(t!=="openai"&&t!=="anthropic"&&t!=="deepseek")throw Error(`Unsupported AI provider: ${t}`);if(!n)throw Error(`Invalid model ref: ${e}. Expected provider:model.`);return{provider:t,model:n}}function ke(e,t){return Qe(t)[e]??e}function be(e){let t=Qe(e);return Object.entries(t).map(([r,n])=>{let i=Te(n);return{alias:r,model_ref:n,provider:i.provider,model:i.model,default:r==="default",capabilities:Jr[i.provider]}})}function Ze(e,t=process.env){return Object.keys(Je).map((r)=>{let n=Qr(e,r),i=Boolean(t[n.api_key_env]);return{provider:r,api_key_env:n.api_key_env,configured:i,source:i?"env":"missing",base_url:n.base_url??null,default_model:n.default_model}})}function et(e,t=process.env){return{default_model:ke("default",e),providers:Ze(e,t),models:be(e)}}function tt(e,t,r=process.env){let n=Ze(t,r).find((i)=>i.provider===e);if(!n)throw Error(`Unsupported AI provider: ${e}`);if(!n.configured)throw Error(`Missing ${n.api_key_env} for ${e}. Set the env var to use this provider.`);return n}import{createHash as Zr,randomUUID as en}from"crypto";var rt=[{kind:"schema",prefix:"schemas/",description:"Machine-readable agent schemas and source rules."},{kind:"index",prefix:"indexes/",description:"Small orientation indexes and future shard manifests."},{kind:"log",prefix:"logs/",description:"Append-only JSONL run and wiki-maintenance log partitions."},{kind:"run",prefix:"runs/",description:"Prompt/tool/cost ledgers and generated output records."},{kind:"wiki_page",prefix:"wiki/",description:"Generated cited Markdown pages, not raw source files."},{kind:"export",prefix:"exports/",description:"Portable exports and snapshots of derived knowledge state."}];function nt(e){let t=typeof e==="string"?Buffer.from(e):Buffer.from(e);return{hash:`sha256:${Zr("sha256").update(t).digest("hex")}`,size_bytes:t.byteLength}}function it(e){return rt.find((r)=>e.startsWith(r.prefix))?.kind??"artifact"}function st(e,t,r="global"){let n=we(e,t),i=e.storage.s3??null,s=i?.prefix?.replace(/^\/+|\/+$/g,"")??"",f=i?`s3://${i.bucket}/${s?`${s}/`:""}`:"";return{scope:r,mode:e.mode,storage_type:e.storage.type,workspace_home:t.home,local_layout:{app_path:z,config_path:t.configPath,json_store_path:t.jsonStorePath,knowledge_db_path:t.knowledgeDbPath,directories:{artifacts:t.artifactsDir,cache:t.cacheDir,exports:t.exportsDir,indexes:t.indexesDir,logs:t.logsDir,runs:t.runsDir,schemas:t.schemasDir,wiki:t.wikiDir}},artifact_store:{type:e.storage.type,artifacts_root:e.storage.artifacts_root,uri_prefix:e.storage.type==="s3"?f:`file://${t.artifactsDir}/`,s3:i?{bucket:i.bucket,prefix:s,region:i.region??null,profile:i.profile??null,server_side_encryption:i.server_side_encryption??null,kms_key_configured:Boolean(i.kms_key_id)}:null},source_ownership:{owner:"open-files",preferred_ref:e.sources.preferred_ref,allowed_schemes:e.sources.allowed_schemes,raw_source_bytes_stored_in_open_knowledge:!1,stores:["source refs","source revisions and hashes","citation spans","redacted extracted chunks","embeddings","generated wiki artifacts","indexes","run ledgers"],does_not_store:["raw open-files bytes","S3 object credentials","connector secrets","hosted tenant ownership state"]},generated_artifacts:rt,scalability:{catalog:"knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.",indexes:"Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.",logs:"Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.",markdown:"Markdown pages are the readable wiki layer over DB/object-store state."},warnings:n.warnings}}function we(e,t){let r=[],n=[];if(!t.home.endsWith(z))n.push(`Workspace home does not end with ${z}: ${t.home}`);if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)r.push("storage.s3.bucket is required when storage.type is s3.");if(!e.storage.s3?.prefix)n.push("storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.");if(e.mode==="local")n.push("storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.")}if(e.storage.type==="local"&&e.storage.s3)n.push("storage.s3 is configured but ignored while storage.type is local.");if(e.sources.preferred_ref!=="open-files")n.push("sources.preferred_ref should stay open-files for durable company knowledge.");if(!e.sources.allowed_schemes.includes("open-files"))r.push("sources.allowed_schemes must include open-files.");return{ok:r.length===0,errors:r,warnings:n}}function ot(e,t,r=new Date){let n=r.toISOString(),i=e.prepare(`
|
|
268
|
+
INSERT INTO storage_objects (
|
|
269
|
+
id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
|
|
270
|
+
)
|
|
271
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
272
|
+
ON CONFLICT(artifact_uri) DO UPDATE SET
|
|
273
|
+
kind = excluded.kind,
|
|
274
|
+
content_type = excluded.content_type,
|
|
275
|
+
hash = excluded.hash,
|
|
276
|
+
size_bytes = excluded.size_bytes,
|
|
277
|
+
metadata_json = excluded.metadata_json,
|
|
278
|
+
updated_at = excluded.updated_at
|
|
279
|
+
`);e.transaction((f)=>{for(let _ of f)i.run(en(),_.uri,_.kind,_.content_type??null,_.hash??null,_.size_bytes??null,JSON.stringify({key:_.key,..._.metadata??{}}),n,n)})(t)}function tn(e){let t=String(e.getUTCFullYear()),r=String(e.getUTCMonth()+1).padStart(2,"0"),n=String(e.getUTCDate()).padStart(2,"0");return{year:t,month:r,day:n}}function rn(){return`# Knowledge Agent Schema v1
|
|
268
280
|
|
|
269
281
|
## Source Rules
|
|
270
282
|
|
|
@@ -289,7 +301,7 @@ VALUES (3, datetime('now'));
|
|
|
289
301
|
## Lint Rules
|
|
290
302
|
|
|
291
303
|
- Flag stale pages, missing citations, contradictions, orphan pages, duplicate pages, and unresolved source refs.
|
|
292
|
-
`}function
|
|
304
|
+
`}function nn(){return`# Knowledge Index
|
|
293
305
|
|
|
294
306
|
This is a compact orientation index for agents. It is not the full search index.
|
|
295
307
|
|
|
@@ -304,13 +316,13 @@ This is a compact orientation index for agents. It is not the full search index.
|
|
|
304
316
|
|
|
305
317
|
Raw source files are resolved through open-files. This app stores source refs,
|
|
306
318
|
citations, chunks, generated wiki artifacts, indexes, and run records.
|
|
307
|
-
`}function
|
|
319
|
+
`}function sn(){return`# Wiki
|
|
308
320
|
|
|
309
321
|
Generated durable knowledge pages live here.
|
|
310
322
|
|
|
311
323
|
Pages should be concise, cited, and organized for both humans and agents.
|
|
312
|
-
`}async function
|
|
313
|
-
`,content_type:"application/x-ndjson"}
|
|
324
|
+
`}async function at(e,t=new Date){let{year:r,month:n,day:i}=tn(t),s="schemas/v1.md",f="indexes/root.md",_="wiki/README.md",a=`logs/${r}/${n}/${i}.jsonl`,o={ts:t.toISOString(),event:"wiki_layout_initialized",schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md"},c=[{key:"schemas/v1.md",body:rn(),content_type:"text/markdown"},{key:"indexes/root.md",body:nn(),content_type:"text/markdown"},{key:"wiki/README.md",body:sn(),content_type:"text/markdown"},{key:a,body:`${JSON.stringify(o)}
|
|
325
|
+
`,content_type:"application/x-ndjson"}],u=await Promise.all(c.map(async(l)=>{let d=await e.put(l);return{key:d.key,uri:d.uri,kind:it(l.key),content_type:l.content_type,...nt(l.body)}}));return{schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md",log_key:a,artifacts:u,written:["schemas/v1.md","indexes/root.md","wiki/README.md",a]}}class ct{options;ensuredWorkspace;cachedConfig;constructor(e={}){this.options=e}get scope(){return this.options.scope??"global"}get workspace(){return this.ensuredWorkspace??xe(this.options.scope,this.options.cwd)}ensureWorkspace(){if(!this.ensuredWorkspace)this.ensuredWorkspace=Re(this.workspace.home);return this.ensuredWorkspace}jsonStorePath(){return this.ensureWorkspace().jsonStorePath}config(){if(!this.cachedConfig){let e=this.ensureWorkspace();this.cachedConfig=ve(e.configPath)}return this.cachedConfig}safetyPolicy(){return Me(this.config(),this.ensureWorkspace())}artifactStore(){return Ue(this.config(),this.ensureWorkspace())}storageContract(){return st(this.config(),this.ensureWorkspace(),this.scope)}validateStorage(){return we(this.config(),this.ensureWorkspace())}paths(){let e=this.ensureWorkspace();return{ok:!0,scope:this.scope,home:e.home,config_path:e.configPath,json_store_path:e.jsonStorePath,knowledge_db_path:e.knowledgeDbPath,artifacts_dir:e.artifactsDir,indexes_dir:e.indexesDir,logs_dir:e.logsDir,runs_dir:e.runsDir,schemas_dir:e.schemasDir,wiki_dir:e.wikiDir,config:this.config(),message:e.home}}initDb(){return C(this.ensureWorkspace().knowledgeDbPath)}dbStats(){let e=this.ensureWorkspace();return C(e.knowledgeDbPath),Le(e.knowledgeDbPath)}async initWiki(){let e=this.ensureWorkspace();C(e.knowledgeDbPath);let t=await at(this.artifactStore()),r=A(e.knowledgeDbPath);try{ot(r,t.artifacts)}finally{r.close()}return t}async ingestManifest(e){let t=this.ensureWorkspace();return He({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}async ingestSource(e,t){let r=this.ensureWorkspace();return Ge({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t,config:this.config(),safetyPolicy:this.safetyPolicy()})}async resolveSource(e,t={}){let r=this.ensureWorkspace();return se({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t.purpose,limit:t.limit,safetyPolicy:this.safetyPolicy()})}async consumeOutbox(e){let t=this.ensureWorkspace();return ze({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}providerStatus(e=process.env){return et(this.config(),e)}modelRegistry(){return be(this.config())}}function ut(e={}){return new ct(e)}var q={name:"@hasna/knowledge",version:"0.2.12",description:"Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",type:"module",bin:{"open-knowledge":"bin/open-knowledge.js","open-knowledge-mcp":"bin/open-knowledge-mcp.js"},files:["bin","src","docs","LICENSE","README.md"],scripts:{test:"bun test","test:cli":"bun test tests/cli.test.ts",build:"bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",prepublishOnly:"bun run build",postinstall:"bun run build"},keywords:["knowledge","cli","agents","json","notes","local","store"],license:"Apache-2.0",publishConfig:{registry:"https://registry.npmjs.org",access:"public"},repository:{type:"git",url:"git+https://github.com/hasna/knowledge.git"},bugs:{url:"https://github.com/hasna/knowledge/issues"},author:"Hasna Inc. <hasna@example.com>",engines:{bun:">=1.0",node:">=18"},dependencies:{"@aws-sdk/client-s3":"^3.1063.0","@aws-sdk/credential-providers":"^3.1063.0","@ai-sdk/anthropic":"^3.0.81","@ai-sdk/deepseek":"^2.0.35","@ai-sdk/openai":"^3.0.68","@modelcontextprotocol/sdk":"^1.29.0",ai:"^6.0.197",zod:"^4.3.6"},devDependencies:{"@types/bun":"^1.3.14"}};var dt={debug:0,info:1,warn:2,error:3},an=()=>{if(process.env.DEBUG)return"debug";if(process.env.LOG_LEVEL==="debug")return"debug";if(process.env.LOG_LEVEL==="warn")return"warn";if(process.env.LOG_LEVEL==="error")return"error";return"info"};function F(e,t,r){if(dt[e]<dt[an()])return;let n={debug:"[DEBUG]",info:"[INFO]",warn:"[WARN]",error:"[ERROR]"}[e],i=r?`${n} ${t} ${JSON.stringify(r)}`:`${n} ${t}`;if(e==="error")console.error(i);else console.error(i)}var cn=["add","list","get","delete","update","archive","restore","upsert","untag","export","prune","dedupe","stats","paths","storage","db","wiki","source","ingest","reindex","providers","safety","help"],lt={ls:"list",rm:"delete",edit:"update",unarchive:"restore"};function un(e){let t=[],r={};for(let n=0;n<e.length;n+=1){let i=e[n];if(!i.startsWith("-")){t.push(i);continue}switch(i){case"--json":r.json=!0;break;case"--yes":case"-y":r.yes=!0;break;case"--help":case"-h":r.help=!0;break;case"--version":case"-v":r.version=!0;break;case"--desc":r.desc=!0;break;case"--page":case"-p":r.page=Number(e[n+1]),n+=1;break;case"--limit":case"-l":r.limit=Number(e[n+1]),n+=1;break;case"--search":case"-s":r.search=e[n+1],n+=1;break;case"--sort":r.sort=e[n+1],n+=1;break;case"--id":r.id=e[n+1],n+=1;break;case"--store":r.store=e[n+1],n+=1;break;case"--title":r.title=e[n+1],n+=1;break;case"--content":r.content=e[n+1],n+=1;break;case"--url":r.url=e[n+1],n+=1;break;case"--tag":case"-t":r.tag=e[n+1],n+=1;break;case"--format":r.format=e[n+1],n+=1;break;case"--completions":r.completions=e[n+1],n+=1;break;case"--purpose":r.purpose=e[n+1],n+=1;break;case"--no-color":r.noColor=!0;break;case"--scope":r.scope=e[n+1],n+=1;break;case"--older-than":r.olderThan=Number(e[n+1]),n+=1;break;case"--empty":r.empty=!0;break;case"--archived":r.archived=!0;break;case"--include-archived":r.includeArchived=!0;break;default:throw Error(`Unknown flag: ${i}. Run 'open-knowledge --help' for valid options.`)}}return{positional:t,flags:r}}function dn(e){if(!e)return"";return lt[e]??e}function ln(e,t){let r=Array.from({length:e.length+1},()=>Array(t.length+1).fill(0));for(let n=0;n<=e.length;n+=1)r[n][0]=n;for(let n=0;n<=t.length;n+=1)r[0][n]=n;for(let n=1;n<=e.length;n+=1)for(let i=1;i<=t.length;i+=1){let s=e[n-1]===t[i-1]?0:1;r[n][i]=Math.min(r[n-1][i]+1,r[n][i-1]+1,r[n-1][i-1]+s)}return r[e.length][t.length]}function fn(e){if(!e)return"";let t=[...cn,...Object.keys(lt)],r="",n=Number.POSITIVE_INFINITY;for(let i of t){let s=ln(e,i);if(s<n)n=s,r=i}return n<=3?r:""}function _n(){console.log(`open-knowledge - local agent knowledge store
|
|
314
326
|
|
|
315
327
|
Usage:
|
|
316
328
|
open-knowledge <command> [options]
|
|
@@ -330,6 +342,7 @@ Commands:
|
|
|
330
342
|
dedupe Remove duplicate items by title+content (requires --yes)
|
|
331
343
|
stats Show knowledge base statistics
|
|
332
344
|
paths Show resolved workspace/store paths
|
|
345
|
+
storage status|validate Inspect local/S3 artifact storage contract
|
|
333
346
|
db init|stats Initialize or inspect local knowledge.db
|
|
334
347
|
wiki init Initialize scalable wiki/schema/index/log artifacts
|
|
335
348
|
source resolve <source-ref> Resolve read-only source content and citation evidence
|
|
@@ -380,5 +393,5 @@ Export Options:
|
|
|
380
393
|
|
|
381
394
|
Prune Options:
|
|
382
395
|
--older-than <days> Remove items older than N days
|
|
383
|
-
--empty Remove items with empty content`)}function
|
|
384
|
-
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(o==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=tn(t[0]);if(!n||r.help||n==="help"){on(t[1]);return}let i=rt({scope:r.scope}),s=r.store;if(!s)if(r.scope==="project"||r.scope==="local")s=i.jsonStorePath();else s=le();if(n==="paths"){h(i.paths(),r.json);return}if(n==="db"){let o=t[1]??"init";if(o!=="init"&&o!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(o==="init"){let c=i.initDb();h({ok:!0,...c,message:`Initialized ${c.path}`},r.json);return}let a=i.dbStats();h({ok:!0,path:i.workspace.knowledgeDbPath,...a,message:`knowledge.db schema v${a.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let a=await i.initWiki();h({ok:!0,...a,message:`Initialized wiki layout in ${i.workspace.home}`},r.json);return}if(n==="safety"){let o=t[1]??"status",a=i.ensureWorkspace(),c=i.safetyPolicy();i.initDb();let u=C(a.knowledgeDbPath);try{if(o==="status"){h({ok:!0,mode:c.mode,workspace:a.home,allow_write_roots:c.allowWriteRoots,read_only_source_access:c.readOnlySourceAccess,network:c.network,redaction:c.redaction,approvals:c.approvals,message:`Safety policy: ${c.mode}`},r.json);return}if(o==="check"){let f=t[2]??"generated_write",d=t[3]??null,y;try{if(f==="web_search")Z(c),y={action:f,target_uri:d,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!d)throw Error("safety check s3_read requires an s3:// target.");K(d,c),y={action:f,target_uri:d,approval_required:!1,approved:!0,decision:"allow"}}else y=Me(u,c,f,d);w(u,{event_type:"safety_check",action:f,target_uri:d,decision:y.decision==="allow"?"allow":"requires_approval",metadata:y}),h({ok:!0,...y,message:`Safety check ${y.decision}`},r.json);return}catch(T){throw w(u,{event_type:"safety_check",action:f,target_uri:d,decision:"deny",metadata:{error:T instanceof Error?T.message:String(T)}}),T}}if(o==="approve"){let f=t[2]??"generated_write",d=t[3]??null,y=Xe(u,{action:f,target_uri:d,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});w(u,{event_type:"approval",action:f,target_uri:d,decision:"allow",metadata:{approval_id:y.id}}),h({ok:!0,...y,action:f,target_uri:d,message:`Approved ${f}`},r.json);return}if(o==="audit"){let f=u.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((d)=>({id:d.id,event_type:d.event_type,action:d.action,target_uri:d.target_uri,decision:d.decision,metadata:JSON.parse(d.metadata_json),created_at:d.created_at}));h({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(o==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let d=ee(f,c);if(d.findings.length>0)te(u,{source_uri:"safety://redact",findings:d.findings,metadata:{command:"safety redact"}});w(u,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:d.findings.length>0?"redacted":"allow",metadata:{findings:d.findings.length}}),h({ok:!0,text:d.text,findings:d.findings,message:`Redacted ${d.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{u.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let a=t[2];if(!a)throw Error("Usage: open-knowledge source resolve <source-ref>");let c=await i.resolveSource(a,{purpose:r.purpose,limit:r.limit});h({ok:!0,...c,message:c.resolved?`Resolved ${c.source_ref} (${c.content.chunks_returned}/${c.content.chunks_total} chunks)`:`Source not indexed: ${a}`},r.json);return}if(n==="ingest"){let o=t[1]??"";if(o==="manifest"){let a=t[2];if(!a)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let c=await i.ingestManifest(a);h({ok:!0,...c,message:`Ingested ${c.items_seen} manifest item(s)`},r.json);return}if(o==="source"){let a=t[2];if(!a)throw Error("Usage: open-knowledge ingest source <source-ref>");let c=await i.ingestSource(a,r.purpose);h({ok:!0,...c,message:`Ingested source ${c.source_ref} (${c.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let a=t[2];if(!a)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let c=await i.consumeOutbox(a);h({ok:!0,...c,message:`Consumed ${c.events_seen} outbox event(s)`},r.json);return}if(n==="providers"){let o=t[1]??"status";if(o==="status"){let a=i.providerStatus(),c=a.providers.filter((u)=>u.configured).length;h({ok:!0,...a,message:`${c}/${a.providers.length} provider credential(s) configured`},r.json);return}if(o==="models"){let a=i.modelRegistry();h({ok:!0,models:a,message:`${a.length} model alias(es)`},r.json);return}if(o==="check"){let a=t[2]??"default",c=me(a,i.config()),u=Te(c),f=Ze(u.provider,i.config());h({ok:!0,target:a,model_ref:c,provider:u.provider,model:u.model,credential:f,message:`${u.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(fe(s),n==="add"){let o=t[1],a=t[2];if(!o||!a)throw Error("Usage: open-knowledge add <title> <content>");v(s,()=>{let c=x(s),u={id:_e(),title:o,content:a,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};c.items.push(u),D(s,c),F("info","Item added",{id:u.id,title:u.title}),h({ok:!0,item:u,message:`Added ${u.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");v(s,()=>{let o=x(s),a=Number.isFinite(r.page)&&r.page>0?r.page:1,c=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,u=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",d=r.format==="table"||!r.json&&!r.format&&an(r),y=r.json||r.format==="json",T=o.items;if(r.archived)T=T.filter((m)=>m.archived===!0);else if(!r.includeArchived)T=T.filter((m)=>!m.archived);if(u)T=T.filter((m)=>m.title.toLowerCase().includes(u)||m.content.toLowerCase().includes(u));if(f)T=T.filter((m)=>m.tags&&m.tags.map((ae)=>ae.toLowerCase()).includes(f));let{sorted:p,sort:L,direction:k}=cn(T,r),R=(a-1)*c,g=p.slice(R,R+c),O=Math.max(1,Math.ceil(p.length/c));if(y){h({ok:!0,page:a,limit:c,total:p.length,total_pages:O,sort:L,direction:k,items:g},!0);return}if(g.length===0){h(`No items found (search=${u||"none"}, tag=${f||"none"})`,!1);return}if(d){let m=(P)=>P,ae=`${m("ID")} ${m("TITLE")} ${m("CREATED")} ${m("URL")} ${m("TAGS")}`;console.log(ae);for(let P of g)console.log(`${P.id} ${m(P.title)} ${P.created_at} ${P.url?m(P.url):""} ${P.tags?.length?m(`[${P.tags.join(", ")}]`):""}`);console.log(`Page ${a}/${O} | showing ${g.length} of ${p.length} | sort=${L} ${k} | search=${u||"none"} | tag=${f||"none"}`)}else{for(let m of g)console.log(`${m.id} ${m.title} ${m.created_at}${m.url?` ${m.url}`:""}${m.tags?.length?` [${m.tags.join(", ")}]`:""}`);console.log(`Page ${a}/${O} | showing ${g.length} of ${p.length} | sort=${L} ${k} | search=${u||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){q(r),v(s,()=>{let a=x(s).items.find((c)=>c.id===r.id||c.short_id===r.id);if(!a)throw Error(`Item not found: ${r.id}`);h({ok:!0,item:a,message:`${a.id}: ${a.title}`},r.json)});return}if(n==="update"){q(r),v(s,()=>{let o=x(s),a=o.items.findIndex((u)=>u.id===r.id||u.short_id===r.id);if(a===-1)throw Error(`Item not found: ${r.id}`);let c=o.items[a];if(r.title!==void 0)c.title=r.title;if(r.content!==void 0)c.content=r.content;if(r.url!==void 0)c.url=r.url;if(r.tag!==void 0){if(c.tags=c.tags||[],!c.tags.map((u)=>u.toLowerCase()).includes(r.tag.toLowerCase()))c.tags.push(r.tag)}c.updated_at=new Date().toISOString(),o.items[a]=c,D(s,o),h({ok:!0,item:c,message:`Updated ${c.id}`},r.json)});return}if(n==="archive"||n==="restore"){q(r),v(s,()=>{let o=x(s),a=o.items.findIndex((u)=>u.id===r.id||u.short_id===r.id);if(a===-1)throw Error(`Item not found: ${r.id}`);let c=o.items[a];c.archived=n==="archive",c.updated_at=new Date().toISOString(),o.items[a]=c,D(s,o),h({ok:!0,item:c,message:`${n==="archive"?"Archived":"Restored"} ${c.id}`},r.json)});return}if(n==="untag"){if(q(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");v(s,()=>{let o=x(s),a=o.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(a===-1)throw Error(`Item not found: ${r.id}`);let c=o.items[a],u=c.tags?.length??0;c.tags=(c.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),c.updated_at=new Date().toISOString(),o.items[a]=c,D(s,o),h({ok:!0,item:c,removed:u-c.tags.length,message:`Removed tag from ${c.id}`},r.json)});return}if(n==="upsert"){let o=r.title??t[1],a=r.content??t[2];v(s,()=>{let c=x(s),u=r.id?c.items.findIndex((y)=>y.id===r.id||y.short_id===r.id):-1,f=new Date().toISOString();if(u===-1){if(!o||!a)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let y=r.id??_e(),T={id:y,short_id:Oe(y),title:o,content:a,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};c.items.push(T),D(s,c),h({ok:!0,created:!0,item:T,message:`Upserted ${T.id}`},r.json);return}let d=c.items[u];if(o!==void 0)d.title=o;if(a!==void 0)d.content=a;if(r.url!==void 0)d.url=r.url;if(r.tag!==void 0){if(d.tags=d.tags||[],!d.tags.map((y)=>y.toLowerCase()).includes(r.tag.toLowerCase()))d.tags.push(r.tag)}d.updated_at=f,c.items[u]=d,D(s,c),h({ok:!0,created:!1,item:d,message:`Upserted ${d.id}`},r.json)});return}if(n==="delete"){if(q(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");v(s,()=>{let o=x(s),a=o.items.length;o.items=o.items.filter((u)=>u.id!==r.id&&u.short_id!==r.id);let c=a!==o.items.length;if(D(s,o),!c)throw Error(`Item not found: ${r.id}`);F("info","Item deleted",{id:r.id}),h({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let o=r.format??"json";if(o!=="json"&&o!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");v(s,()=>{let a=x(s);if(o==="jsonl")for(let c of a.items)console.log(JSON.stringify(c));else h({ok:!0,items:a.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");v(s,()=>{let o=x(s),a=o.items.length;if(r.olderThan!==void 0){let u=new Date;u.setDate(u.getDate()-r.olderThan),o.items=o.items.filter((f)=>new Date(f.created_at)>=u)}if(r.empty)o.items=o.items.filter((u)=>u.content.trim().length>0);let c=a-o.items.length;D(s,o),F("info","Prune completed",{pruned:c,remaining:o.items.length}),h({ok:!0,pruned:c,remaining:o.items.length,message:`Pruned ${c} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");v(s,()=>{let o=x(s),a=new Set,c=o.items.length;o.items=o.items.filter((f)=>{let d=`${f.title}\x00${f.content}`;if(a.has(d))return!1;return a.add(d),!0});let u=c-o.items.length;D(s,o),F("info","Dedupe completed",{removed:u,remaining:o.items.length}),h({ok:!0,removed:u,remaining:o.items.length,message:`Dedupe removed ${u} duplicate(s)`},r.json)});return}if(n==="stats"){v(s,()=>{let o=x(s),a=o.items.filter((k)=>!k.archived),c=a.length,u=o.items.length-c,f=a.filter((k)=>k.url).length,d=a.filter((k)=>k.tags&&k.tags.length>0).length,y=c>0?a.map((k)=>k.created_at).sort()[0]:null,T=c>0?a.map((k)=>k.created_at).sort()[c-1]:null,p={};for(let k of a)for(let R of k.tags||[])p[R]=(p[R]||0)+1;let L=Object.entries(p).sort((k,R)=>R[1]-k[1]).slice(0,5).map(([k,R])=>({tag:k,count:R}));h({ok:!0,total:c,archived:u,with_url:f,with_tags:d,oldest:y,newest:T,top_tags:L,message:`${c} items | ${f} with URL | ${d} with tags`},r.json)});return}let l=nn(t[0]),_=l?` Did you mean '${l}'?`:"";throw F("warn","Unknown command",{input:t[0],suggestion:l}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)un(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);F("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{nn as suggestCommand,cn as sortItems,un as run,en as parseArgs};
|
|
396
|
+
--empty Remove items with empty content`)}function pn(e){if(e==="add"){console.log("Usage: open-knowledge add <title> <content> [--url <url>] [-t <tag>] [--json]");return}if(e==="list"||e==="ls"){console.log("Usage: open-knowledge list|ls [--format table|json] [-p <page>] [-l <limit>] [-s <search>] [-t <tag>] [--sort created|title] [--desc] [--json]");return}if(e==="get"){console.log("Usage: open-knowledge get --id <id> [--json]");return}if(e==="update"||e==="edit"){console.log("Usage: open-knowledge update|edit --id <id> [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="archive"){console.log("Usage: open-knowledge archive --id <id> [--json]");return}if(e==="restore"||e==="unarchive"){console.log("Usage: open-knowledge restore|unarchive --id <id> [--json]");return}if(e==="upsert"){console.log("Usage: open-knowledge upsert [title] [content] [--id <id>] [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="untag"){console.log("Usage: open-knowledge untag --id <id> -t <tag> [--json]");return}if(e==="delete"||e==="rm"){console.log("Usage: open-knowledge delete|rm --id <id> -y [--json]");return}if(e==="export"){console.log("Usage: open-knowledge export [--format jsonl] [--json]");return}if(e==="prune"){console.log("Usage: open-knowledge prune --yes [--older-than <days>] [--empty] [--json]");return}if(e==="dedupe"){console.log("Usage: open-knowledge dedupe --yes [--json]");return}if(e==="stats"){console.log("Usage: open-knowledge stats [--json]");return}if(e==="paths"){console.log("Usage: open-knowledge paths [--scope local|global|project] [--json]");return}if(e==="storage"){console.log("Usage: open-knowledge storage status|validate [--scope local|global|project] [--json]");return}if(e==="db"){console.log("Usage: open-knowledge db init|stats [--scope local|global|project] [--json]");return}if(e==="wiki"){console.log("Usage: open-knowledge wiki init [--scope local|global|project] [--json]");return}if(e==="source"){console.log("Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]");return}if(e==="ingest"){console.log("Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]");return}if(e==="reindex"){console.log("Usage: open-knowledge reindex outbox <file|s3://bucket/key> [--scope local|global|project] [--json]");return}if(e==="providers"){console.log("Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]");return}if(e==="safety"){console.log("Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]");return}_n()}function gn(e){if(e.noColor||process.env.NO_COLOR)return!1;if(process.env.FORCE_COLOR)return!0;return process.stdout.isTTY===!0}function E(e,t,r){if(t){console.log(JSON.stringify(e,null,2));return}if(typeof e==="string"){console.log(e);return}console.log(e.message??JSON.stringify(e,null,2))}function Y(e){if(!e.id)throw Error("Missing required --id. Example: open-knowledge get --id <id>")}function hn(e,t){let r=t.sort??"created";if(r!=="created"&&r!=="title")throw Error("Invalid --sort value. Use 'created' or 'title'.");let n=[...e].sort((i,s)=>{if(r==="title")return i.title.localeCompare(s.title);return i.created_at.localeCompare(s.created_at)});if(t.desc)n.reverse();return{sorted:n,sort:r,direction:t.desc?"desc":"asc"}}async function En(e){let{positional:t,flags:r}=un(e);if(F("debug","CLI invoked",{command:t[0],flags:{json:r.json,store:r.store}}),r.version){console.log(r.json?JSON.stringify({name:q.name,version:q.version},null,2):`${q.name} ${q.version}`);return}if(r.completions){let a=r.completions;if(a==="bash")console.log('_open_knowledge() { local cur; cur="${COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge');else if(a==="zsh")console.log(`#compdef open-knowledge
|
|
397
|
+
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex providers safety help ls rm edit unarchive"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=dn(t[0]);if(!n||r.help||n==="help"){pn(t[1]);return}let i=ut({scope:r.scope}),s=r.store;if(!s)if(r.scope==="project"||r.scope==="local")s=i.jsonStorePath();else s=fe();if(n==="paths"){E(i.paths(),r.json);return}if(n==="storage"){let a=t[1]??"status";if(a==="status"){let o=i.storageContract(),c=i.validateStorage();E({ok:c.ok,...o,validation:c,message:`${o.storage_type} artifact storage at ${o.artifact_store.uri_prefix}`},r.json);return}if(a==="validate"){let o=i.validateStorage();E({ok:o.ok,validation:o,message:o.ok?"Storage contract valid":`Storage contract invalid: ${o.errors.join("; ")}`},r.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(n==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let c=i.initDb();E({ok:!0,...c,message:`Initialized ${c.path}`},r.json);return}let o=i.dbStats();E({ok:!0,path:i.workspace.knowledgeDbPath,...o,message:`knowledge.db schema v${o.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let o=await i.initWiki();E({ok:!0,...o,message:`Initialized wiki layout in ${i.workspace.home}`},r.json);return}if(n==="safety"){let a=t[1]??"status",o=i.ensureWorkspace(),c=i.safetyPolicy();i.initDb();let u=A(o.knowledgeDbPath);try{if(a==="status"){E({ok:!0,mode:c.mode,workspace:o.home,allow_write_roots:c.allowWriteRoots,read_only_source_access:c.readOnlySourceAccess,network:c.network,redaction:c.redaction,approvals:c.approvals,message:`Safety policy: ${c.mode}`},r.json);return}if(a==="check"){let l=t[2]??"generated_write",d=t[3]??null,y;try{if(l==="web_search")ee(c),y={action:l,target_uri:d,approval_required:!1,approved:!0,decision:"allow"};else if(l==="s3_read"){if(!d)throw Error("safety check s3_read requires an s3:// target.");j(d,c),y={action:l,target_uri:d,approval_required:!1,approved:!0,decision:"allow"}}else y=$e(u,c,l,d);w(u,{event_type:"safety_check",action:l,target_uri:d,decision:y.decision==="allow"?"allow":"requires_approval",metadata:y}),E({ok:!0,...y,message:`Safety check ${y.decision}`},r.json);return}catch(m){throw w(u,{event_type:"safety_check",action:l,target_uri:d,decision:"deny",metadata:{error:m instanceof Error?m.message:String(m)}}),m}}if(a==="approve"){let l=t[2]??"generated_write",d=t[3]??null,y=We(u,{action:l,target_uri:d,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});w(u,{event_type:"approval",action:l,target_uri:d,decision:"allow",metadata:{approval_id:y.id}}),E({ok:!0,...y,action:l,target_uri:d,message:`Approved ${l}`},r.json);return}if(a==="audit"){let l=u.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((d)=>({id:d.id,event_type:d.event_type,action:d.action,target_uri:d.target_uri,decision:d.decision,metadata:JSON.parse(d.metadata_json),created_at:d.created_at}));E({ok:!0,events:l,message:`${l.length} audit event(s)`},r.json);return}if(a==="redact"){let l=t.slice(2).join(" ");if(!l)throw Error("Usage: open-knowledge safety redact <text>");let d=te(l,c);if(d.findings.length>0)re(u,{source_uri:"safety://redact",findings:d.findings,metadata:{command:"safety redact"}});w(u,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:d.findings.length>0?"redacted":"allow",metadata:{findings:d.findings.length}}),E({ok:!0,text:d.text,findings:d.findings,message:`Redacted ${d.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{u.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let o=t[2];if(!o)throw Error("Usage: open-knowledge source resolve <source-ref>");let c=await i.resolveSource(o,{purpose:r.purpose,limit:r.limit});E({ok:!0,...c,message:c.resolved?`Resolved ${c.source_ref} (${c.content.chunks_returned}/${c.content.chunks_total} chunks)`:`Source not indexed: ${o}`},r.json);return}if(n==="ingest"){let a=t[1]??"";if(a==="manifest"){let o=t[2];if(!o)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let c=await i.ingestManifest(o);E({ok:!0,...c,message:`Ingested ${c.items_seen} manifest item(s)`},r.json);return}if(a==="source"){let o=t[2];if(!o)throw Error("Usage: open-knowledge ingest source <source-ref>");let c=await i.ingestSource(o,r.purpose);E({ok:!0,...c,message:`Ingested source ${c.source_ref} (${c.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){if((t[1]??"")!=="outbox")throw Error("Invalid reindex action. Use 'outbox'.");let o=t[2];if(!o)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let c=await i.consumeOutbox(o);E({ok:!0,...c,message:`Consumed ${c.events_seen} outbox event(s)`},r.json);return}if(n==="providers"){let a=t[1]??"status";if(a==="status"){let o=i.providerStatus(),c=o.providers.filter((u)=>u.configured).length;E({ok:!0,...o,message:`${c}/${o.providers.length} provider credential(s) configured`},r.json);return}if(a==="models"){let o=i.modelRegistry();E({ok:!0,models:o,message:`${o.length} model alias(es)`},r.json);return}if(a==="check"){let o=t[2]??"default",c=ke(o,i.config()),u=Te(c),l=tt(u.provider,i.config());E({ok:!0,target:o,model_ref:c,provider:u.provider,model:u.model,credential:l,message:`${u.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(_e(s),n==="add"){let a=t[1],o=t[2];if(!a||!o)throw Error("Usage: open-knowledge add <title> <content>");v(s,()=>{let c=x(s),u={id:pe(),title:a,content:o,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};c.items.push(u),U(s,c),F("info","Item added",{id:u.id,title:u.title}),E({ok:!0,item:u,message:`Added ${u.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");v(s,()=>{let a=x(s),o=Number.isFinite(r.page)&&r.page>0?r.page:1,c=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,u=r.search?String(r.search).toLowerCase():"",l=r.tag?String(r.tag).toLowerCase():"",d=r.format==="table"||!r.json&&!r.format&&gn(r),y=r.json||r.format==="json",m=a.items;if(r.archived)m=m.filter((T)=>T.archived===!0);else if(!r.includeArchived)m=m.filter((T)=>!T.archived);if(u)m=m.filter((T)=>T.title.toLowerCase().includes(u)||T.content.toLowerCase().includes(u));if(l)m=m.filter((T)=>T.tags&&T.tags.map((ce)=>ce.toLowerCase()).includes(l));let{sorted:p,sort:I,direction:k}=hn(m,r),S=(o-1)*c,g=p.slice(S,S+c),O=Math.max(1,Math.ceil(p.length/c));if(y){E({ok:!0,page:o,limit:c,total:p.length,total_pages:O,sort:I,direction:k,items:g},!0);return}if(g.length===0){E(`No items found (search=${u||"none"}, tag=${l||"none"})`,!1);return}if(d){let T=(P)=>P,ce=`${T("ID")} ${T("TITLE")} ${T("CREATED")} ${T("URL")} ${T("TAGS")}`;console.log(ce);for(let P of g)console.log(`${P.id} ${T(P.title)} ${P.created_at} ${P.url?T(P.url):""} ${P.tags?.length?T(`[${P.tags.join(", ")}]`):""}`);console.log(`Page ${o}/${O} | showing ${g.length} of ${p.length} | sort=${I} ${k} | search=${u||"none"} | tag=${l||"none"}`)}else{for(let T of g)console.log(`${T.id} ${T.title} ${T.created_at}${T.url?` ${T.url}`:""}${T.tags?.length?` [${T.tags.join(", ")}]`:""}`);console.log(`Page ${o}/${O} | showing ${g.length} of ${p.length} | sort=${I} ${k} | search=${u||"none"} | tag=${l||"none"}`)}});return}if(n==="get"){Y(r),v(s,()=>{let o=x(s).items.find((c)=>c.id===r.id||c.short_id===r.id);if(!o)throw Error(`Item not found: ${r.id}`);E({ok:!0,item:o,message:`${o.id}: ${o.title}`},r.json)});return}if(n==="update"){Y(r),v(s,()=>{let a=x(s),o=a.items.findIndex((u)=>u.id===r.id||u.short_id===r.id);if(o===-1)throw Error(`Item not found: ${r.id}`);let c=a.items[o];if(r.title!==void 0)c.title=r.title;if(r.content!==void 0)c.content=r.content;if(r.url!==void 0)c.url=r.url;if(r.tag!==void 0){if(c.tags=c.tags||[],!c.tags.map((u)=>u.toLowerCase()).includes(r.tag.toLowerCase()))c.tags.push(r.tag)}c.updated_at=new Date().toISOString(),a.items[o]=c,U(s,a),E({ok:!0,item:c,message:`Updated ${c.id}`},r.json)});return}if(n==="archive"||n==="restore"){Y(r),v(s,()=>{let a=x(s),o=a.items.findIndex((u)=>u.id===r.id||u.short_id===r.id);if(o===-1)throw Error(`Item not found: ${r.id}`);let c=a.items[o];c.archived=n==="archive",c.updated_at=new Date().toISOString(),a.items[o]=c,U(s,a),E({ok:!0,item:c,message:`${n==="archive"?"Archived":"Restored"} ${c.id}`},r.json)});return}if(n==="untag"){if(Y(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");v(s,()=>{let a=x(s),o=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(o===-1)throw Error(`Item not found: ${r.id}`);let c=a.items[o],u=c.tags?.length??0;c.tags=(c.tags??[]).filter((l)=>l.toLowerCase()!==r.tag.toLowerCase()),c.updated_at=new Date().toISOString(),a.items[o]=c,U(s,a),E({ok:!0,item:c,removed:u-c.tags.length,message:`Removed tag from ${c.id}`},r.json)});return}if(n==="upsert"){let a=r.title??t[1],o=r.content??t[2];v(s,()=>{let c=x(s),u=r.id?c.items.findIndex((y)=>y.id===r.id||y.short_id===r.id):-1,l=new Date().toISOString();if(u===-1){if(!a||!o)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let y=r.id??pe(),m={id:y,short_id:Ae(y),title:a,content:o,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:l,updated_at:l};c.items.push(m),U(s,c),E({ok:!0,created:!0,item:m,message:`Upserted ${m.id}`},r.json);return}let d=c.items[u];if(a!==void 0)d.title=a;if(o!==void 0)d.content=o;if(r.url!==void 0)d.url=r.url;if(r.tag!==void 0){if(d.tags=d.tags||[],!d.tags.map((y)=>y.toLowerCase()).includes(r.tag.toLowerCase()))d.tags.push(r.tag)}d.updated_at=l,c.items[u]=d,U(s,c),E({ok:!0,created:!1,item:d,message:`Upserted ${d.id}`},r.json)});return}if(n==="delete"){if(Y(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");v(s,()=>{let a=x(s),o=a.items.length;a.items=a.items.filter((u)=>u.id!==r.id&&u.short_id!==r.id);let c=o!==a.items.length;if(U(s,a),!c)throw Error(`Item not found: ${r.id}`);F("info","Item deleted",{id:r.id}),E({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let a=r.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");v(s,()=>{let o=x(s);if(a==="jsonl")for(let c of o.items)console.log(JSON.stringify(c));else E({ok:!0,items:o.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");v(s,()=>{let a=x(s),o=a.items.length;if(r.olderThan!==void 0){let u=new Date;u.setDate(u.getDate()-r.olderThan),a.items=a.items.filter((l)=>new Date(l.created_at)>=u)}if(r.empty)a.items=a.items.filter((u)=>u.content.trim().length>0);let c=o-a.items.length;U(s,a),F("info","Prune completed",{pruned:c,remaining:a.items.length}),E({ok:!0,pruned:c,remaining:a.items.length,message:`Pruned ${c} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");v(s,()=>{let a=x(s),o=new Set,c=a.items.length;a.items=a.items.filter((l)=>{let d=`${l.title}\x00${l.content}`;if(o.has(d))return!1;return o.add(d),!0});let u=c-a.items.length;U(s,a),F("info","Dedupe completed",{removed:u,remaining:a.items.length}),E({ok:!0,removed:u,remaining:a.items.length,message:`Dedupe removed ${u} duplicate(s)`},r.json)});return}if(n==="stats"){v(s,()=>{let a=x(s),o=a.items.filter((k)=>!k.archived),c=o.length,u=a.items.length-c,l=o.filter((k)=>k.url).length,d=o.filter((k)=>k.tags&&k.tags.length>0).length,y=c>0?o.map((k)=>k.created_at).sort()[0]:null,m=c>0?o.map((k)=>k.created_at).sort()[c-1]:null,p={};for(let k of o)for(let S of k.tags||[])p[S]=(p[S]||0)+1;let I=Object.entries(p).sort((k,S)=>S[1]-k[1]).slice(0,5).map(([k,S])=>({tag:k,count:S}));E({ok:!0,total:c,archived:u,with_url:l,with_tags:d,oldest:y,newest:m,top_tags:I,message:`${c} items | ${l} with URL | ${d} with tags`},r.json)});return}let f=fn(t[0]),_=f?` Did you mean '${f}'?`:"";throw F("warn","Unknown command",{input:t[0],suggestion:f}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)En(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);F("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{fn as suggestCommand,hn as sortItems,En as run,un as parseArgs};
|
|
@@ -133,6 +133,18 @@ Raw files still route through `open-files`. Knowledge S3 storage is for derived
|
|
|
133
133
|
artifacts such as wiki pages, index shards, schema versions, logs, exports, and
|
|
134
134
|
run outputs.
|
|
135
135
|
|
|
136
|
+
The storage contract is inspectable through:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
open-knowledge storage status --scope project --json
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
That contract names the local app path, SQLite catalog, generated artifact
|
|
143
|
+
classes, S3 bucket/prefix when configured, and the source ownership rule that
|
|
144
|
+
raw source bytes stay in `open-files`. The `storage_objects` table catalogs
|
|
145
|
+
generated artifacts by URI, kind, hash, size, and metadata so local mode and
|
|
146
|
+
remote/S3 mode share the same DB-facing shape.
|
|
147
|
+
|
|
136
148
|
## Wiki Model
|
|
137
149
|
|
|
138
150
|
The Karpathy-style wiki pattern is implemented as scalable artifacts, not three
|