@hasna/knowledge 0.2.20 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/bin/open-knowledge-mcp.js +345 -14
- package/bin/open-knowledge.js +82 -70
- package/docs/architecture/ai-native-knowledge-base.md +8 -0
- package/package.json +1 -1
- package/src/auth.ts +123 -0
- package/src/cli.ts +109 -4
- package/src/remote-client.ts +268 -0
- package/src/service.ts +98 -0
- package/src/storage-contract.ts +28 -0
- package/src/workspace.ts +11 -0
package/bin/open-knowledge.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
// @bun
|
|
3
|
-
var O=import.meta.require;import{readFileSync as
|
|
4
|
-
`);return t}function
|
|
3
|
+
var O=import.meta.require;import{readFileSync as pe,writeFileSync as fe,existsSync as ge,renameSync as zn,unlinkSync as Et}from"fs";import{randomUUID as yt}from"crypto";import{existsSync as Wn,mkdirSync as je,readFileSync as Xn,writeFileSync as _t}from"fs";import{homedir as ft}from"os";import{dirname as $n,join as C,resolve as qn}from"path";var se=C(".hasna","apps","knowledge");function We(){return C(ft(),".open-knowledge","db.json")}function Xe(){return C(ft(),".hasna","apps","knowledge")}function Hn(e=process.cwd()){return qn(e,se)}function ie(e){return{home:e,configPath:C(e,"config.json"),jsonStorePath:C(e,"db.json"),knowledgeDbPath:C(e,"knowledge.db"),artifactsDir:C(e,"artifacts"),cacheDir:C(e,"cache"),exportsDir:C(e,"exports"),indexesDir:C(e,"indexes"),logsDir:C(e,"logs"),runsDir:C(e,"runs"),schemasDir:C(e,"schemas"),wikiDir:C(e,"wiki")}}function Bn(){return{version:1,mode:"local",hosted:{api_url:"https://knowledge.hasna.xyz"},storage:{type:"local",artifacts_root:"artifacts"},sources:{preferred_ref:"open-files",allowed_schemes:["open-files","s3","file","https","http"]},providers:{default_model:"openai:gpt-5.2",aliases:{fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"},openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},embeddings:{default_model:"openai:text-embedding-3-small",dimensions:1536,batch_size:64,max_parallel_calls:4},safety:{network:{web_search_enabled:!1,s3_reads_enabled:!1,allowed_s3_buckets:[]},redaction:{enabled:!0},approvals:{generated_writes_require_approval:!0}}}}function gt(e){let t=ie(e);je(t.home,{recursive:!0});for(let n of[t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir])je(n,{recursive:!0});if(!Wn(t.configPath))_t(t.configPath,`${JSON.stringify(Bn(),null,2)}
|
|
4
|
+
`);return t}function pt(e,t=process.cwd()){if(e==="project"||e==="local")return ie(Hn(t));return ie(Xe())}function oe(e){je($n(e),{recursive:!0})}function mt(e){let t=Xn(e,"utf8");return JSON.parse(t)}function ht(e,t){oe(e),_t(e,`${JSON.stringify(t,null,2)}
|
|
5
|
+
`)}function $e(){return ie(Xe()).jsonStorePath}function qe(e){if(!ge(e))if(oe(e),e===$e()&&ge(We()))fe(e,pe(We(),"utf8"));else fe(e,JSON.stringify({items:[]},null,2))}function Jn(e){return`${e}.lock`}function Gn(e,t){let i=Date.now();while(Date.now()-i<5000){try{if(!ge(e)){fe(e,JSON.stringify({owner:t,ts:Date.now()}));return}let d=JSON.parse(pe(e,"utf8"));if(Date.now()-d.ts>1e4)Et(e)}catch{}let s=Date.now();while(Date.now()-s<50);}throw Error(`Could not acquire lock on ${e} after 5000ms`)}function Yn(e,t){try{if(ge(e)){if(JSON.parse(pe(e,"utf8")).owner===t)Et(e)}}catch{}}function P(e){qe(e);let t=pe(e,"utf8"),n=JSON.parse(t);if(!n||!Array.isArray(n.items))return{items:[]};return n}function j(e,t){let n=`${e}.tmp.${yt()}`;fe(n,JSON.stringify(t,null,2)),zn(n,e)}function D(e,t){let n=yt(),r=Jn(e);Gn(r,n);try{return t()}finally{Yn(r,n)}}function He(){return`k_${Date.now().toString(36)}_${Math.random().toString(36).slice(2,8)}`}function kt(e){return e.replace(/^k_/,"").slice(0,12)}import{Database as Vn}from"bun:sqlite";var Qn=`
|
|
5
6
|
PRAGMA journal_mode = WAL;
|
|
6
7
|
PRAGMA foreign_keys = ON;
|
|
7
8
|
|
|
@@ -168,7 +169,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
168
169
|
|
|
169
170
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
170
171
|
VALUES (1, datetime('now'));
|
|
171
|
-
`,
|
|
172
|
+
`,Zn=`
|
|
172
173
|
DROP TABLE IF EXISTS chunks_fts;
|
|
173
174
|
|
|
174
175
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
@@ -181,7 +182,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
|
181
182
|
|
|
182
183
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
183
184
|
VALUES (2, datetime('now'));
|
|
184
|
-
`,
|
|
185
|
+
`,er=`
|
|
185
186
|
CREATE TABLE IF NOT EXISTS audit_events (
|
|
186
187
|
id TEXT PRIMARY KEY,
|
|
187
188
|
event_type TEXT NOT NULL,
|
|
@@ -212,7 +213,7 @@ CREATE INDEX IF NOT EXISTS idx_approval_gates_status ON approval_gates(status);
|
|
|
212
213
|
|
|
213
214
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
214
215
|
VALUES (3, datetime('now'));
|
|
215
|
-
`,
|
|
216
|
+
`,tr=`
|
|
216
217
|
CREATE TABLE IF NOT EXISTS vector_index_entries (
|
|
217
218
|
id TEXT PRIMARY KEY,
|
|
218
219
|
chunk_id TEXT NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
|
|
@@ -243,7 +244,7 @@ CREATE INDEX IF NOT EXISTS idx_vector_index_status ON vector_index_entries(statu
|
|
|
243
244
|
|
|
244
245
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
245
246
|
VALUES (4, datetime('now'));
|
|
246
|
-
`,
|
|
247
|
+
`,nr=`
|
|
247
248
|
CREATE TABLE IF NOT EXISTS reindex_queue (
|
|
248
249
|
id TEXT PRIMARY KEY,
|
|
249
250
|
kind TEXT NOT NULL,
|
|
@@ -264,8 +265,9 @@ CREATE INDEX IF NOT EXISTS idx_reindex_queue_source_uri ON reindex_queue(source_
|
|
|
264
265
|
|
|
265
266
|
INSERT OR IGNORE INTO schema_versions(version, applied_at)
|
|
266
267
|
VALUES (5, datetime('now'));
|
|
267
|
-
`;function b(e){
|
|
268
|
-
|
|
268
|
+
`;function b(e){oe(e);let t=new Vn(e);return t.exec("PRAGMA foreign_keys = ON;"),t.exec("PRAGMA busy_timeout = 5000;"),t}function S(e){let t=b(e);try{if(t.exec(Qn),Z(t)<2)t.exec(Zn);if(Z(t)<3)t.exec(er);if(Z(t)<4)t.exec(tr);if(Z(t)<5)t.exec(nr);return{path:e,schema_version:Z(t)}}finally{t.close()}}function Z(e){return e.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0}function I(e,t){return e.query(`SELECT COUNT(*) AS n FROM ${t}`).get()?.n??0}function bt(e){let t=b(e);try{return{schema_version:Z(t),sources:I(t,"sources"),source_revisions:I(t,"source_revisions"),chunks:I(t,"chunks"),wiki_pages:I(t,"wiki_pages"),citations:I(t,"citations"),indexes:I(t,"knowledge_indexes"),runs:I(t,"runs"),run_events:I(t,"run_events"),redaction_findings:I(t,"redaction_findings"),audit_events:I(t,"audit_events"),approval_gates:I(t,"approval_gates"),storage_objects:I(t,"storage_objects"),embeddings:I(t,"chunk_embeddings"),vector_entries:I(t,"vector_index_entries"),reindex_queue:I(t,"reindex_queue")}}finally{t.close()}}import{existsSync as rr,mkdirSync as vt,readFileSync as ir,writeFileSync as sr}from"fs";import{dirname as or,join as Be,relative as ar,sep as ur}from"path";function ae(e){let t=e.replace(/\\/g,"/").trim();if(!t||t.startsWith("/"))throw Error(`Invalid artifact key: ${e}`);let n=t.split("/").filter(Boolean);if(n.length===0||n.some((r)=>r==="."||r===".."))throw Error(`Invalid artifact key: ${e}`);return n.join("/")}function ze(e,t){let n=ar(e,t);if(n.startsWith("..")||n===".."||n.startsWith(`..${ur}`))throw Error(`Artifact path escapes root: ${t}`)}class wt{root;type="local";canRead=!0;canWrite=!0;constructor(e){this.root=e;vt(e,{recursive:!0})}async put(e){let t=ae(e.key),n=Be(this.root,t);return ze(this.root,n),vt(or(n),{recursive:!0}),sr(n,e.body),{key:t,uri:`file://${n}`}}async getText(e){let t=ae(e),n=Be(this.root,t);return ze(this.root,n),ir(n,"utf8")}async exists(e){let t=ae(e),n=Be(this.root,t);return ze(this.root,n),rr(n)}}class Tt{options;type="s3";canRead=!0;canWrite=!0;client;constructor(e){this.options=e;this.client=e.client}async getClient(){if(this.client)return this.client;let[{S3Client:e},{fromIni:t}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]);return this.client=new e({region:this.options.region,credentials:this.options.profile?t({profile:this.options.profile}):void 0,maxAttempts:this.options.max_attempts}),this.client}objectKey(e){let t=ae(e),n=this.options.prefix?ae(this.options.prefix):"";return n?`${n}/${t}`:t}async put(e){let[{PutObjectCommand:t},n]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),r=this.objectKey(e.key);return await n.send(new t({Bucket:this.options.bucket,Key:r,Body:e.body,ContentType:e.content_type,Metadata:e.metadata,ServerSideEncryption:this.options.server_side_encryption,SSEKMSKeyId:this.options.kms_key_id})),{key:r,uri:`s3://${this.options.bucket}/${r}`}}async getText(e){let[{GetObjectCommand:t},n]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),r=this.objectKey(e),i=await n.send(new t({Bucket:this.options.bucket,Key:r}));if(!i.Body)return"";return await i.Body.transformToString()}async exists(e){let[{HeadObjectCommand:t},n]=await Promise.all([import("@aws-sdk/client-s3"),this.getClient()]),r=this.objectKey(e);try{return await n.send(new t({Bucket:this.options.bucket,Key:r})),!0}catch(i){let s=i instanceof Error?i.name:"";if(s==="NotFound"||s==="NoSuchKey"||s==="NotFoundError")return!1;throw i}}}function St(e,t){if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)throw Error("S3 artifact storage requires storage.s3.bucket");return new Tt({bucket:e.storage.s3.bucket,prefix:e.storage.s3.prefix,region:e.storage.s3.region,profile:e.storage.s3.profile,max_attempts:e.storage.s3.max_attempts,server_side_encryption:e.storage.s3.server_side_encryption,kms_key_id:e.storage.s3.kms_key_id})}return new wt(t.artifactsDir)}import{existsSync as dr,mkdirSync as cr,readFileSync as lr,unlinkSync as _r,writeFileSync as fr}from"fs";import{homedir as gr}from"os";import{dirname as pr,join as Rt}from"path";var Je="https://knowledge.hasna.xyz";function X(e){let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")throw Error("Knowledge API URL must use http or https.");let n=t.pathname.replace(/\/+$/,"");if(n==="/api"||n==="/api/v1")t.pathname="/";else if(n.endsWith("/api/v1"))t.pathname=n.slice(0,-7)||"/";else if(n.endsWith("/api"))t.pathname=n.slice(0,-4)||"/";return t.toString().replace(/\/+$/,"")}function he(e=process.env){if(e.HASNA_KNOWLEDGE_AUTH_PATH)return e.HASNA_KNOWLEDGE_AUTH_PATH;let t=e.HASNA_KNOWLEDGE_AUTH_DIR??Rt(gr(),".hasna","knowledge");return Rt(t,"auth.json")}function me(e,t=process.env){return X(t.KNOWLEDGE_API_URL??e?.hosted?.api_url??Je)}function xt(e=process.env){try{let t=he(e);if(!dr(t))return null;let n=JSON.parse(lr(t,"utf8"));return typeof n.api_key==="string"&&n.api_key.length>0?n:null}catch{return null}}function Ot(e,t=process.env){let n=he(t),r={...e,api_url:e.api_url?X(e.api_url):void 0,created_at:e.created_at??new Date().toISOString()};return cr(pr(n),{recursive:!0,mode:448}),fr(n,`${JSON.stringify(r,null,2)}
|
|
269
|
+
`,{mode:384}),r}function Nt(e=process.env){try{return _r(he(e)),!0}catch{return!1}}function Ge(e=process.env){if(e.KNOWLEDGE_API_KEY)return{apiKey:e.KNOWLEDGE_API_KEY,source:"env"};if(e.HASNA_KNOWLEDGE_API_KEY)return{apiKey:e.HASNA_KNOWLEDGE_API_KEY,source:"env"};let t=xt(e);return t?.api_key?{apiKey:t.api_key,source:"file"}:{apiKey:null,source:"none"}}function At(e,t=process.env){let n=xt(t),r=Ge(t),i=t.KNOWLEDGE_API_URL?me(e,t):n?.api_url?X(n.api_url):me(e,t);return{authenticated:Boolean(r.apiKey),source:r.source,api_url:i,auth_path:he(t),email:r.source==="file"?n?.email??null:null,org_id:r.source==="file"?n?.org_id??null:null,org_slug:r.source==="file"?n?.org_slug??null:null,user_id:r.source==="file"?n?.user_id??null:null,api_key_present:Boolean(r.apiKey)}}import{randomUUID as en}from"crypto";import{randomUUID as mr}from"crypto";var Ye={openai:{api_key_env:"OPENAI_API_KEY",default_model:"gpt-5.2"},anthropic:{api_key_env:"ANTHROPIC_API_KEY",default_model:"claude-sonnet-4-6"},deepseek:{api_key_env:"DEEPSEEK_API_KEY",default_model:"deepseek-chat"}},hr={openai:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!0,reasoning:!0,embeddings:!0},anthropic:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!0,native_web_search:!1,reasoning:!0,embeddings:!1},deepseek:{text_generation:!0,structured_output:!0,tool_usage:!0,tool_streaming:!0,image_input:!1,native_web_search:!1,reasoning:!0,embeddings:!1}},Er={default:"openai:gpt-5.2",fast:"openai:gpt-5-mini",reasoning:"anthropic:claude-opus-4-6",sonnet:"anthropic:claude-sonnet-4-6",deepseek:"deepseek:deepseek-chat","deepseek-reasoning":"deepseek:deepseek-reasoner"};function Lt(e){return e?.providers??{}}function $(e,t){let n=Lt(e)[t]??{};return{...Ye[t],...n}}function Ct(e){let t=Lt(e);return{...Er,...t.default_model?{default:t.default_model}:{},...t.aliases??{}}}function L(e){let[t,...n]=e.split(":"),r=n.join(":");if(t!=="openai"&&t!=="anthropic"&&t!=="deepseek")throw Error(`Unsupported AI provider: ${t}`);if(!r)throw Error(`Invalid model ref: ${e}. Expected provider:model.`);return{provider:t,model:r}}function q(e,t){return Ct(t)[e]??e}function Ve(e){let t=Ct(e);return Object.entries(t).map(([n,r])=>{let i=L(r);return{alias:n,model_ref:r,provider:i.provider,model:i.model,default:n==="default",capabilities:hr[i.provider]}})}function Pt(e,t=process.env){return Object.keys(Ye).map((n)=>{let r=$(e,n),i=Boolean(t[r.api_key_env]);return{provider:n,api_key_env:r.api_key_env,configured:i,source:i?"env":"missing",base_url:r.base_url??null,default_model:r.default_model}})}function Dt(e,t=process.env){return{default_model:q("default",e),providers:Pt(e,t),models:Ve(e)}}function J(e,t,n=process.env){let r=Pt(t,n).find((i)=>i.provider===e);if(!r)throw Error(`Unsupported AI provider: ${e}`);if(!r.configured)throw Error(`Missing ${r.api_key_env} for ${e}. Set the env var to use this provider.`);return r}async function yr(e){if(e==="openai"){let{createOpenAI:n}=await import("@ai-sdk/openai");return n}if(e==="anthropic"){let{createAnthropic:n}=await import("@ai-sdk/anthropic");return n}let{createDeepSeek:t}=await import("@ai-sdk/deepseek");return t}async function kr(e={}){let{createProviderRegistry:t}=await import("ai"),n=e.env??process.env,r={};for(let i of Object.keys(Ye)){let s=$(e.config,i),d=n[s.api_key_env];if(!d)continue;let c=e.factories?.[i]??await yr(i);r[i]=c({apiKey:d,baseURL:s.base_url})}return t(r)}async function Ut(e,t={}){let n=q(e,t.config),r=L(n);return J(r.provider,t.config,t.env),(await kr(t)).languageModel(n)}function It(e,t){for(let n of t){let r=e[n];if(typeof r==="number"&&Number.isFinite(r))return r}return 0}function Ee(e){let t=e.usage??{};return{provider:e.provider,model:e.model,input_tokens:It(t,["inputTokens","promptTokens","input_tokens","prompt_tokens"]),output_tokens:It(t,["outputTokens","completionTokens","output_tokens","completion_tokens"]),cost_usd:e.costUsd??0,metadata:{usage:t,provider_metadata:e.providerMetadata??{}}}}function ye(e,t){let n=`usage_${mr()}`;return e.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
270
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,t.run_id??null,t.provider,t.model,t.input_tokens,t.output_tokens,t.cost_usd,JSON.stringify(t.metadata),t.created_at??new Date().toISOString()]),n}import{createHash as Wr}from"crypto";function Qe(e){return["deleted","stale","invalidated","reindex_required"].includes((e??"").toLowerCase())}function H(e){let t=e.status??null;return{source_owner:"open-files",source_ref:e.source_ref??null,source_uri:e.source_uri??null,source_kind:e.source_kind??null,source_revision_id:e.source_revision_id??null,revision:e.revision??null,hash:e.hash??null,chunk_id:e.chunk_id??null,start_offset:e.start_offset??null,end_offset:e.end_offset??null,status:t,read_only:!0,citation_required:!0,resolver:e.resolver??null,stale:Qe(t)}}function Ze(e){return{source_owner:"open-files",generated_from:e.generated_from,artifact_key:e.artifact_key,source_refs:e.source_refs??[],read_only_sources:!0,citation_required:e.citation_required??!0,raw_source_bytes_stored_in_open_knowledge:!1}}function Kt(e,t){return{...e,provenance:t}}import{createHash as jt}from"crypto";var br="openai:text-embedding-3-small",Wt=1536;function ke(e){return e?.embeddings??{}}function Mt(e,t){return`${e}_${jt("sha256").update(t).digest("hex").slice(0,20)}`}function tt(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function M(e,t){for(let n of t){let r=e[n];if(typeof r==="string"&&r.length>0)return r}return null}function Ft(e,t){for(let n of t){let r=e[n];if(typeof r==="number"&&Number.isFinite(r))return r}return null}function et(e){return Math.sqrt(e.reduce((t,n)=>t+n*n,0))}function vr(e,t,n=et(t)){let r=et(e);if(r===0||n===0)return 0;let i=Math.min(e.length,t.length),s=0;for(let d=0;d<i;d+=1)s+=e[d]*t[d];return s/(r*n)}function wr(e,t){let n=jt("sha256").update(e).digest();return Array.from({length:t},(r,i)=>{let s=n[i%n.length]/255;return Number((s*2-1).toFixed(6))})}async function Tr(e,t,n=process.env){J("openai",t,n);let r=$(t,"openai"),{createOpenAI:i}=await import("@ai-sdk/openai"),s=i({apiKey:n[r.api_key_env],baseURL:r.base_url});if(s.embeddingModel)return s.embeddingModel(e);if(s.textEmbedding)return s.textEmbedding(e);if(s.textEmbeddingModel)return s.textEmbeddingModel(e);throw Error("OpenAI provider does not expose an embedding model factory.")}function G(e,t){if(!e||e==="default"||e==="embedding")return ke(t).default_model??br;return e}async function Xt(e,t={}){let n=G(t.modelRef,t.config),r=L(n);if(r.provider!=="openai")throw Error(`Embedding provider ${r.provider} is not supported yet. Use openai:text-embedding-3-small.`);let i=t.dimensions??ke(t.config).dimensions??Wt;if(t.fake)return{provider:r.provider,model:r.model,dimensions:i,vectors:e.map((a)=>wr(a,i)),usage:{input_tokens:e.reduce((a,u)=>a+Math.max(1,Math.ceil(u.split(/\s+/).filter(Boolean).length*1.25)),0)}};let{embedMany:s}=await import("ai"),d=await Tr(r.model,t.config,t.env),c=await s({model:d,values:e,maxParallelCalls:t.maxParallelCalls??ke(t.config).max_parallel_calls,providerOptions:{openai:{dimensions:i}}}),_=c.embeddings;return{provider:r.provider,model:r.model,dimensions:_[0]?.length??i,vectors:_,usage:{input_tokens:c.usage?.tokens??0}}}function Sr(e,t){if(t.sourceRevisionId)return e.query(`SELECT
|
|
269
271
|
c.id,
|
|
270
272
|
c.text,
|
|
271
273
|
c.token_count,
|
|
@@ -303,7 +305,7 @@ VALUES (5, datetime('now'));
|
|
|
303
305
|
ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
|
|
304
306
|
WHERE v.id IS NULL
|
|
305
307
|
ORDER BY c.created_at ASC, c.ordinal ASC
|
|
306
|
-
LIMIT ?`).all(t.provider,t.model,t.limit)}function
|
|
308
|
+
LIMIT ?`).all(t.provider,t.model,t.limit)}function Rr(e){let t=tt(e.metadata_json),n=t.provenance;if(n&&typeof n==="object"&&!Array.isArray(n))return n;return H({source_ref:M(t,["source_ref"]),source_uri:e.source_uri??M(t,["source_uri"]),source_kind:e.source_kind??M(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??M(t,["revision"]),hash:e.hash??M(t,["hash"]),chunk_id:e.id,start_offset:e.start_offset??Ft(t,["start_offset"]),end_offset:e.end_offset??Ft(t,["end_offset"]),status:M(t,["status"]),resolver:"open-files-read-only"})}function xr(e,t,n,r){let i=e.prepare(`
|
|
307
309
|
INSERT INTO chunk_embeddings (id, chunk_id, provider, model, dimensions, vector_json, created_at)
|
|
308
310
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
309
311
|
ON CONFLICT(chunk_id, provider, model) DO UPDATE SET
|
|
@@ -332,10 +334,10 @@ VALUES (5, datetime('now'));
|
|
|
332
334
|
status = excluded.status,
|
|
333
335
|
metadata_json = excluded.metadata_json,
|
|
334
336
|
updated_at = excluded.updated_at
|
|
335
|
-
`);return e.transaction(()=>{for(let
|
|
337
|
+
`);return e.transaction(()=>{for(let c=0;c<t.length;c+=1){let _=t[c],a=n.vectors[c];if(!a)continue;let u=tt(_.metadata_json),o=Rr(_),l=o.source_ref??M(u,["source_ref"]),f=o.source_uri??_.source_uri??M(u,["source_uri"]),g=o.revision??_.revision??M(u,["revision"]),E=o.hash??_.hash??M(u,["hash"]),h=o.status??M(u,["status"])??"active",m=JSON.stringify(a);i.run(Mt("emb",`${_.id}\x00${n.provider}\x00${n.model}`),_.id,n.provider,n.model,n.dimensions,m,r),s.run(Mt("vec",`${_.id}\x00${n.provider}\x00${n.model}`),_.id,_.source_revision_id,n.provider,n.model,n.dimensions,m,et(a),f,l,g,E,o.start_offset,o.end_offset,_.token_count,h,JSON.stringify({...u,provenance:o,embedded_at:r}),r,r)}})(),t.length}async function be(e){let t=G(e.modelRef,e.config),n=L(t);if(n.provider!=="openai")throw Error(`Embedding provider ${n.provider} is not supported yet.`);let r=(e.now??new Date).toISOString(),i=Math.max(1,Math.min(e.limit??100,1000));S(e.dbPath);let s=b(e.dbPath),d;try{d=Sr(s,{provider:n.provider,model:n.model,limit:i,sourceRevisionId:e.sourceRevisionId})}finally{s.close()}if(d.length===0)return{provider:n.provider,model:n.model,dimensions:e.dimensions??ke(e.config).dimensions??Wt,chunks_seen:0,chunks_embedded:0,embeddings_upserted:0,vector_entries_upserted:0,usage:{input_tokens:0}};let c=await Xt(d.map((a)=>a.text),e),_=b(e.dbPath);try{let a=xr(_,d,c,r);return{provider:c.provider,model:c.model,dimensions:c.dimensions,chunks_seen:d.length,chunks_embedded:d.length,embeddings_upserted:a,vector_entries_upserted:a,usage:c.usage}}finally{_.close()}}function $t(e){S(e);let t=b(e);try{let n=t.query("SELECT COUNT(*) AS n FROM chunk_embeddings").get()?.n??0,r=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0,i=t.query(`SELECT provider, model, dimensions, COUNT(*) AS entries, MAX(updated_at) AS updated_at
|
|
336
338
|
FROM vector_index_entries
|
|
337
339
|
GROUP BY provider, model, dimensions
|
|
338
|
-
ORDER BY provider, model`).all();return{total_embeddings:
|
|
340
|
+
ORDER BY provider, model`).all();return{total_embeddings:n,total_vector_entries:r,indexes:i}}finally{t.close()}}async function ve(e){let t=G(e.modelRef,e.config),n=L(t),r=Math.max(1,Math.min(e.limit??10,100)),i=await Xt([e.query],e),s=i.vectors[0]??[];S(e.dbPath);let d=b(e.dbPath);try{let _=d.query(`SELECT
|
|
339
341
|
v.chunk_id,
|
|
340
342
|
c.text,
|
|
341
343
|
v.vector_json,
|
|
@@ -347,7 +349,7 @@ VALUES (5, datetime('now'));
|
|
|
347
349
|
v.metadata_json
|
|
348
350
|
FROM vector_index_entries v
|
|
349
351
|
JOIN chunks c ON c.id = v.chunk_id
|
|
350
|
-
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(
|
|
352
|
+
WHERE v.provider = ? AND v.model = ? AND v.status = 'active'`).all(n.provider,n.model).map((a)=>{let u=JSON.parse(a.vector_json),o=tt(a.metadata_json),l=o.provenance&&typeof o.provenance==="object"&&!Array.isArray(o.provenance)?o.provenance:null;return{chunk_id:a.chunk_id,score:vr(s,u,a.vector_norm),text:a.text,source_uri:a.source_uri,source_ref:a.source_ref,revision:a.revision,hash:a.hash,provenance:l}}).sort((a,u)=>u.score-a.score).slice(0,r);return{provider:n.provider,model:n.model,dimensions:i.dimensions,query:e.query,results:_}}finally{d.close()}}function Te(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function U(e,t){for(let n of t){let r=e[n];if(typeof r==="string"&&r.length>0)return r}return null}function qt(e,t){for(let n of t){let r=e[n];if(typeof r==="number"&&Number.isFinite(r))return r}return null}function Ht(e){return Array.from(new Set(e))}function Or(e){let t=e.normalize("NFKC").toLowerCase().match(/[\p{L}\p{N}_]+/gu)??[];return Ht(t.filter((n)=>n.length>0)).slice(0,16)}function Nr(e){if(e.length===0)return null;return e.map((t)=>`${t}*`).join(" OR ")}function Ar(e){return e.replace(/[\\%_]/g,(t)=>`\\${t}`)}function Bt(e,t){return e.flatMap((n)=>Array.from({length:t},()=>`%${Ar(n)}%`))}function Ir(e,t){let n=Number.isFinite(e)?1/(1+Math.abs(e)):0,r=1/(1+t);return Se(Math.max(n,r))}function zt(e,t){if(t.length===0)return 0;let n=t.filter((r)=>e.includes(r)).length;if(n===0)return 0;return Se(Math.min(0.85,0.35+n/t.length*0.5))}function Lr(e){return Se(Math.max(0,Math.min(1,(e+1)/2)))}function Se(e){return Number(e.toFixed(6))}function ue(e,t){let n=e.keyword??0,r=e.semantic??0,i=e.catalog??0,s=t?.chunk_id?0.05:0;return Se(Math.min(1,n*0.55+r*0.4+i*0.35+s))}function nt(e){let t=e.provenance;return t&&typeof t==="object"&&!Array.isArray(t)?t:null}function Cr(e){let t=Te(e.chunk_metadata_json),n=nt(t);if(n)return n;if(!e.source_revision_id&&!e.source_uri)return null;return H({source_ref:U(t,["source_ref"]),source_uri:e.source_uri??U(t,["source_uri"]),source_kind:e.source_kind??U(t,["source_kind"]),source_revision_id:e.source_revision_id,revision:e.revision??U(t,["revision"]),hash:e.hash??U(t,["hash"]),chunk_id:e.chunk_id,start_offset:e.start_offset??qt(t,["start_offset"]),end_offset:e.end_offset??qt(t,["end_offset"]),status:U(t,["status"]),resolver:"open-files-read-only"})}function Pr(e,t,n){if(!t)return[];return e.query(`SELECT
|
|
351
353
|
chunks_fts.chunk_id,
|
|
352
354
|
c.kind AS chunk_kind,
|
|
353
355
|
c.wiki_page_id,
|
|
@@ -376,96 +378,96 @@ VALUES (5, datetime('now'));
|
|
|
376
378
|
LEFT JOIN wiki_pages wp ON wp.id = c.wiki_page_id
|
|
377
379
|
WHERE chunks_fts MATCH ?
|
|
378
380
|
ORDER BY rank ASC
|
|
379
|
-
LIMIT ?`).all(t,
|
|
381
|
+
LIMIT ?`).all(t,n)}function Jt(e,t){if(t.length===0)return"1 = 0";return t.map(()=>`(${e.map((r)=>`lower(COALESCE(${r}, '')) LIKE ? ESCAPE '\\'`).join(" OR ")})`).join(" OR ")}function Dr(e,t,n){let r=["path","title","artifact_uri","metadata_json"];return e.query(`SELECT id, path, title, artifact_uri, content_hash, status, metadata_json
|
|
380
382
|
FROM wiki_pages
|
|
381
|
-
WHERE status = 'active' AND (${
|
|
383
|
+
WHERE status = 'active' AND (${Jt(r,t)})
|
|
382
384
|
ORDER BY updated_at DESC
|
|
383
|
-
LIMIT ?`).all(...
|
|
385
|
+
LIMIT ?`).all(...Bt(t,r.length),n)}function Ur(e,t,n){let r=["kind","name","shard_key","artifact_uri","metadata_json"];return e.query(`SELECT id, kind, name, artifact_uri, shard_key, metadata_json
|
|
384
386
|
FROM knowledge_indexes
|
|
385
|
-
WHERE ${
|
|
387
|
+
WHERE ${Jt(r,t)}
|
|
386
388
|
ORDER BY updated_at DESC
|
|
387
|
-
LIMIT ?`).all(...
|
|
389
|
+
LIMIT ?`).all(...Bt(t,r.length),n)}function Kr(e,t){let n=Te(e.chunk_metadata_json),r=Cr(e),i=U(n,["source_ref"]),s=e.source_uri??U(n,["source_uri"]),d=Boolean(e.wiki_page_id),c={kind:d?"wiki_chunk":"source_chunk",id:e.chunk_id,title:d?e.wiki_title:e.source_title,text:e.text,score:0,scores:{keyword:t},source:s||i?{uri:s,ref:i,kind:e.source_kind??U(n,["source_kind"]),revision:e.revision??U(n,["revision"]),hash:e.hash??U(n,["hash"])}:null,citation:{chunk_id:e.chunk_id,start_offset:e.start_offset,end_offset:e.end_offset},artifact:d?{uri:e.wiki_artifact_uri,path:e.wiki_path,hash:e.wiki_content_hash,shard_key:e.wiki_path}:null,provenance:r,reasons:["keyword_match"]};return c.score=ue(c.scores,c.citation),c}function Mr(e,t){let n=Te(e.metadata_json),r=zt(`${e.path} ${e.title} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"wiki_page",id:e.id,title:e.title,text:null,score:0,scores:{catalog:r},source:null,citation:null,artifact:{uri:e.artifact_uri,path:e.path,hash:e.content_hash,shard_key:e.path},provenance:nt(n),reasons:["wiki_catalog_match"]};return i.score=ue(i.scores,i.citation),i}function Fr(e,t){let n=Te(e.metadata_json),r=zt(`${e.kind} ${e.name} ${e.shard_key??""} ${e.artifact_uri??""} ${e.metadata_json}`.toLowerCase(),t),i={kind:"knowledge_index",id:e.id,title:e.name,text:null,score:0,scores:{catalog:r},source:null,citation:null,artifact:{uri:e.artifact_uri,path:U(n,["artifact_key"]),hash:U(n,["content_hash"]),shard_key:e.shard_key},provenance:nt(n),reasons:["index_catalog_match"]};return i.score=ue(i.scores,i.citation),i}function we(e,t){let n=`${t.kind}:${t.id}`,r=e.get(n);if(!r){e.set(n,t);return}r.scores={keyword:Math.max(r.scores.keyword??0,t.scores.keyword??0)||void 0,semantic:Math.max(r.scores.semantic??0,t.scores.semantic??0)||void 0,catalog:Math.max(r.scores.catalog??0,t.scores.catalog??0)||void 0},r.reasons=Ht([...r.reasons,...t.reasons]),r.text=r.text??t.text,r.title=r.title??t.title,r.source=r.source??t.source,r.citation=r.citation??t.citation,r.artifact=r.artifact??t.artifact,r.provenance=r.provenance??t.provenance,r.score=ue(r.scores,r.citation)}function jr(e){let t={source_chunk:0,wiki_chunk:1,wiki_page:2,knowledge_index:3};return e.sort((n,r)=>{if(r.score!==n.score)return r.score-n.score;return t[n.kind]-t[r.kind]||n.id.localeCompare(r.id)})}async function Re(e){let t=e.query.trim();if(!t)throw Error("Search query is required.");let n=Math.max(1,Math.min(e.limit??10,100)),r=Or(t),i=Nr(r),s=e.semantic===!0||e.fake===!0||Boolean(e.modelRef),d=[],c=null,_=null,a=null,u=0,o=0,l=0,f=new Map;S(e.dbPath);let g=b(e.dbPath);try{let h=Pr(g,i,Math.max(n*3,20));u=h.length,h.forEach((k,p)=>we(f,Kr(k,Ir(k.rank,p))));let m=Dr(g,r,Math.max(n,10)),T=Ur(g,r,Math.max(n,10));o=m.length+T.length,m.forEach((k)=>we(f,Mr(k,r))),T.forEach((k)=>we(f,Fr(k,r)))}finally{g.close()}if(s)try{let h=await ve({dbPath:e.dbPath,query:t,limit:Math.max(n*3,20),config:e.config,env:e.env,modelRef:e.modelRef,dimensions:e.dimensions,fake:e.fake,batchSize:e.batchSize,maxParallelCalls:e.maxParallelCalls});c=h.provider,_=h.model,a=h.dimensions,l=h.results.length;for(let m of h.results){let T={kind:"source_chunk",id:m.chunk_id,title:null,text:m.text,score:0,scores:{semantic:Lr(m.score)},source:{uri:m.source_uri,ref:m.source_ref,kind:m.provenance?.source_kind??null,revision:m.revision,hash:m.hash},citation:{chunk_id:m.chunk_id,start_offset:m.provenance?.start_offset??null,end_offset:m.provenance?.end_offset??null},artifact:null,provenance:m.provenance,reasons:["semantic_match"]};T.score=ue(T.scores,T.citation),we(f,T)}}catch(h){d.push(`semantic_search_failed: ${h instanceof Error?h.message:String(h)}`)}let E=jr(Array.from(f.values())).slice(0,n);return{query:t,limit:n,mode:{keyword:!0,catalog:!0,semantic:s},semantic_provider:c,semantic_model:_,semantic_dimensions:a,counts:{keyword_results:u,catalog_results:o,semantic_results:l,merged_results:E.length},warnings:d,results:E}}function Gt(e,t){return`${e}_${Wr("sha256").update(t).digest("hex").slice(0,20)}`}function Yt(e){return e.normalize("NFKC").trim().replace(/\s+/g," ").toLowerCase()}function Xr(e){return Array.from(new Set(Yt(e).match(/[\p{L}\p{N}_]+/gu)??[])).slice(0,16)}function $r(e){return[e.title,e.text].filter(Boolean).join(" ").toLowerCase()}function qr(e,t){if(t.length===0)return 0;let n=$r(e),r=t.filter((i)=>n.includes(i)).length;return Number((r/t.length).toFixed(6))}function Hr(e){if(!e)return!0;if("read_only"in e)return e.read_only===!0;if("read_only_sources"in e)return e.read_only_sources===!0;return!0}function Vt(e){if(!e)return!1;if("stale"in e&&e.stale)return!0;if("status"in e)return Qe(e.status);return!1}function Br(e){if(Vt(e.provenance))return 0;if(e.source?.hash||e.source?.revision)return 1;if(e.artifact?.hash)return 0.85;if(e.provenance&&"source_refs"in e.provenance&&e.provenance.source_refs.length>0)return 0.75;return 0.55}function zr(e){if(e.citation?.chunk_id&&(e.source?.uri||e.artifact?.uri))return 1;if(e.provenance&&"citation_required"in e.provenance&&e.provenance.citation_required)return 0.75;if(e.artifact?.uri)return 0.65;return 0.35}function Jr(e){if(e.kind==="wiki_chunk")return 0.85;if(e.kind==="source_chunk")return 0.8;if(e.kind==="wiki_page")return 0.65;return 0.55}function Gr(e,t){let n={base_score:e.score,exact_score:qr(e,t),citation_score:zr(e),freshness_score:Br(e),authority_score:Jr(e)},r=Math.min(1,n.base_score*0.65+n.exact_score*0.1+n.citation_score*0.1+n.freshness_score*0.1+n.authority_score*0.05),i=new Set(e.reasons);if(n.exact_score>0.5)i.add("exact_term");if(n.citation_score>=0.75)i.add("cited_source");if(n.freshness_score>=0.85)i.add("fresh_source");return{...e,score:Number(r.toFixed(6)),reasons:Array.from(i),rerank:{...n,final_score:Number(r.toFixed(6))}}}function Qt(e,t){let n=e.text??e.title;if(!n)return null;let r=n.replace(/\s+/g," ").trim();return r.length<=t?r:`${r.slice(0,Math.max(0,t-1)).trim()}...`}function Yr(e){return{id:Gt("cite",`${e.kind}\x00${e.id}\x00${e.source?.uri??""}\x00${e.artifact?.uri??""}`),result_id:e.id,kind:e.kind,source_uri:e.source?.uri??null,source_ref:e.source?.ref??null,artifact_uri:e.artifact?.uri??null,artifact_path:e.artifact?.path??null,revision:e.source?.revision??null,hash:e.source?.hash??e.artifact?.hash??null,chunk_id:e.citation?.chunk_id??null,start_offset:e.citation?.start_offset??null,end_offset:e.citation?.end_offset??null,quote:Qt(e,500),provenance:e.provenance}}function Vr(e,t,n){let r=Qt(e,n);if(!r)return null;return{id:Gt("excerpt",`${e.kind}\x00${e.id}`),result_id:e.id,citation_id:t.id,kind:e.kind,text:r,score:e.score}}function xe(e){return e.map(()=>"?").join(", ")}function Qr(e,t){let n=t.map((c)=>c.citation?.chunk_id).filter((c)=>Boolean(c)),r=t.filter((c)=>c.kind==="wiki_page").map((c)=>c.id),i=[],s=[];if(n.length===0&&r.length===0)return{citations:i,backlinks:s};let d=b(e);try{if(n.length>0)i.push(...d.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
388
390
|
FROM citations
|
|
389
|
-
WHERE chunk_id IN (${
|
|
391
|
+
WHERE chunk_id IN (${xe(n)})
|
|
390
392
|
ORDER BY created_at DESC
|
|
391
|
-
LIMIT 50`).all(...
|
|
393
|
+
LIMIT 50`).all(...n));if(r.length>0)i.push(...d.query(`SELECT id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset
|
|
392
394
|
FROM citations
|
|
393
|
-
WHERE wiki_page_id IN (${
|
|
395
|
+
WHERE wiki_page_id IN (${xe(r)})
|
|
394
396
|
ORDER BY created_at DESC
|
|
395
|
-
LIMIT 50`).all(...
|
|
397
|
+
LIMIT 50`).all(...r)),s.push(...d.query(`SELECT from_page_id, to_page_id, label
|
|
396
398
|
FROM wiki_backlinks
|
|
397
|
-
WHERE from_page_id IN (${
|
|
398
|
-
LIMIT 50`).all(...
|
|
399
|
-
`)}function
|
|
400
|
-
${JSON.stringify(
|
|
401
|
-
${JSON.stringify(
|
|
402
|
-
`)}function
|
|
403
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[t.runId,"knowledge-prompt",t.prompt,t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.now])}finally{
|
|
404
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${
|
|
399
|
+
WHERE from_page_id IN (${xe(r)}) OR to_page_id IN (${xe(r)})
|
|
400
|
+
LIMIT 50`).all(...r,...r))}finally{d.close()}return{citations:i,backlinks:s}}async function Oe(e){let t=Math.max(200,Math.min(e.contextChars??1200,4000)),n=await Re(e),r=Xr(n.query),i=[...n.warnings],s=new Set,d=new Set,_=n.results.filter((o)=>{if(!Hr(o.provenance))return i.push(`permission_filtered: ${o.kind}:${o.id}`),s.add("Dropped a result because provenance was not read-only."),!1;if(Vt(o.provenance))return i.push(`stale_filtered: ${o.kind}:${o.id}`),d.add("Dropped a stale result whose source status requires reindexing."),!1;return!0}).map((o)=>Gr(o,r)).sort((o,l)=>l.score-o.score||o.id.localeCompare(l.id)).slice(0,n.limit),a=_.map(Yr),u=_.map((o,l)=>Vr(o,a[l],t)).filter((o)=>Boolean(o));for(let o of _){if(o.provenance&&"read_only"in o.provenance&&o.provenance.read_only)s.add("All source-backed excerpts are read-only and citation-required.");if(o.rerank.freshness_score>=0.85)d.add("Fresh source revision/hash or artifact hash is present for top context.")}return{query:n.query,normalized_query:Yt(n.query),created_at:new Date().toISOString(),mode:n.mode,warnings:i,search_counts:n.counts,results:_,citations:a,excerpts:u,graph:Qr(e.dbPath,_),notes:{permissions:Array.from(s),freshness:Array.from(d)}}}function rt(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function st(e){return`C${e+1}`}function Zr(e,t){if(t.excerpts.length===0)return`No indexed knowledge matched the prompt: ${e}`;return[`Found ${t.excerpts.length} relevant knowledge excerpt(s) for: ${e}`,"",...t.excerpts.slice(0,5).map((r,i)=>{let s=t.citations.find((c)=>c.id===r.citation_id),d=s?.source_ref??s?.source_uri??s?.artifact_path??s?.artifact_uri??"unknown source";return`[${st(i)}] ${r.text} (${d})`})].join(`
|
|
401
|
+
`)}function ei(e,t){let n=t.citations.map((i,s)=>({id:st(s),source_ref:i.source_ref,source_uri:i.source_uri,artifact_path:i.artifact_path,revision:i.revision,hash:i.hash,quote:i.quote})),r=t.excerpts.map((i,s)=>({id:st(s),kind:i.kind,text:i.text,score:i.score}));return[`Prompt: ${e}`,"","Use only the provided context. Cite claims with citation ids like [C1]. If context is insufficient, say what is missing.","",`Context excerpts:
|
|
402
|
+
${JSON.stringify(r,null,2)}`,"",`Citations:
|
|
403
|
+
${JSON.stringify(n,null,2)}`].join(`
|
|
404
|
+
`)}function ti(e,t){if(t.citations.length===0)return[];return[{kind:"answer_note",title:e.length>80?`${e.slice(0,77)}...`:e,citations:t.citations.map((n)=>n.id),requires_approval:!0}]}function ni(e,t){let n=b(e);try{n.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
405
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[t.runId,"knowledge-prompt",t.prompt,t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.now])}finally{n.close()}}function it(e,t){let n=b(e);try{n.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
406
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${en()}`,t.runId,t.level,t.event,JSON.stringify(t.metadata),t.now])}finally{n.close()}}function Zt(e,t){let n=b(e);try{n.run(`UPDATE runs
|
|
405
407
|
SET status = ?, provider = ?, model = ?, metadata_json = ?, updated_at = ?
|
|
406
|
-
WHERE id = ?`,[t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.runId])}finally{
|
|
408
|
+
WHERE id = ?`,[t.status,t.provider,t.model,JSON.stringify(t.metadata),t.now,t.runId])}finally{n.close()}}function ri(e,t,n,r,i,s,d={}){let c=b(e);try{ye(c,{run_id:t,provider:r,model:i,input_tokens:n.input_tokens,output_tokens:n.output_tokens,cost_usd:n.cost_usd,metadata:d,created_at:s})}finally{c.close()}}async function tn(e){let t=e.prompt.trim();if(!t)throw Error("Knowledge prompt is required.");let n=(e.now??new Date).toISOString(),r=`run_${en()}`,i=q(e.modelRef??"default",e.config),s=L(i);S(e.dbPath),ni(e.dbPath,{runId:r,prompt:t,status:e.generate?"running":"dry_run",provider:e.generate?s.provider:"local",model:e.generate?s.model:"context-draft",metadata:{semantic:e.semantic===!0||e.fake===!0||Boolean(e.modelRef),approve_write:e.approveWrite===!0,generated:e.generate===!0},now:n});let{prompt:d,generate:c,approveWrite:_,now:a,...u}=e,o=await Oe({...u,query:t});it(e.dbPath,{runId:r,level:"info",event:"context_retrieved",metadata:{results:o.results.length,citations:o.citations.length,warnings:o.warnings},now:n});let l=Zr(t,o),f=!1,g="local",E="context-draft",h={input_tokens:rt(t)+o.excerpts.reduce((p,x)=>p+rt(x.text),0),output_tokens:rt(l),cost_usd:0},m=[...o.warnings];if(e.generate)try{if(e.fake)f=!0,g=s.provider,E=s.model,l=`Fake generated answer for: ${t}
|
|
407
409
|
|
|
408
|
-
${l}`;else{let{generateText:p}=await import("ai"),
|
|
409
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[
|
|
410
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${
|
|
411
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
410
|
+
${l}`;else{let{generateText:p}=await import("ai"),x=await Ut(i,{config:e.config,env:e.env}),N=await p({model:x,system:"You answer company knowledge-base prompts using only provided context and citation ids.",prompt:ei(t,o)});f=!0,g=s.provider,E=s.model,l=N.text;let v=Ee({provider:g,model:E,usage:N.usage,providerMetadata:N.providerMetadata});h={input_tokens:v.input_tokens,output_tokens:v.output_tokens,cost_usd:v.cost_usd}}}catch(p){throw it(e.dbPath,{runId:r,level:"error",event:"answer_generation_failed",metadata:{message:p instanceof Error?p.message:String(p)},now:n}),Zt(e.dbPath,{runId:r,status:"failed",provider:s.provider,model:s.model,metadata:{generated:!1,error:p instanceof Error?p.message:String(p)},now:n}),p}let T=ti(t,o),k={approved:e.approveWrite===!0,durable_writes_performed:!1,reason:e.approveWrite?"Approval flag recorded; durable wiki writing is deferred to the wiki compile task.":"Dry-run mode: proposed wiki updates require approval before durable writes."};return it(e.dbPath,{runId:r,level:"info",event:f?"answer_generated":"answer_drafted",metadata:{provider:g,model:E,proposed_updates:T.length,durable_writes_performed:!1},now:n}),ri(e.dbPath,r,h,g,E,n,{generated:f,citations:o.citations.length}),Zt(e.dbPath,{runId:r,status:f?"completed":"dry_run",provider:g,model:E,metadata:{generated:f,citations:o.citations.length,proposed_updates:T.length,approve_write:e.approveWrite===!0},now:n}),{run_id:r,prompt:t,generated:f,provider:g,model:E,answer:l,context:o,citations:o.citations,proposed_wiki_updates:T,write_policy:k,usage:h,warnings:m}}import{createHash as pi,randomUUID as mi}from"crypto";import{existsSync as hi,readFileSync as Ei}from"fs";import{basename as yi}from"path";function nn(e,t){if(!e)throw Error(t);return e}function ii(e){let n=e.slice(13).split("/").filter(Boolean),r=n[0];if(r!=="file"&&r!=="source")throw Error("Invalid open-files ref. Expected open-files://file/<id>, open-files://file/<id>/revision/<revision_id>, or open-files://source/<id>/path/<path>.");let i=nn(n[1],"Invalid open-files ref. Missing id.");if(r==="file"){if(n.length===2)return{kind:"open-files",uri:e,entity:r,id:i};if(n[2]==="revision"&&n[3]&&n.length===4)return{kind:"open-files",uri:e,entity:r,id:i,revision_id:decodeURIComponent(n[3])};throw Error("Invalid open-files file ref. Expected open-files://file/<id>/revision/<revision_id>.")}let s=n.indexOf("path"),d=s>=0?decodeURIComponent(n.slice(s+1).join("/")):void 0;return{kind:"open-files",uri:e,entity:r,id:i,path:d}}function si(e){let t=new URL(e),n=nn(t.hostname,"Invalid s3 ref. Missing bucket."),r=decodeURIComponent(t.pathname.replace(/^\/+/,""));if(!r)throw Error("Invalid s3 ref. Missing object key.");return{kind:"s3",uri:e,bucket:n,key:r}}function oi(e){let t=new URL(e);return{kind:"file",uri:e,path:decodeURIComponent(t.pathname)}}function ai(e){let t=new URL(e);return{kind:"web",uri:e,url:t.toString()}}function K(e){if(e.startsWith("open-files://"))return ii(e);if(e.startsWith("s3://"))return si(e);if(e.startsWith("file://"))return oi(e);if(e.startsWith("https://")||e.startsWith("http://"))return ai(e);throw Error(`Unsupported source ref scheme: ${e}`)}function rn(e,t=K(e)){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function sn(e){let t=K(e);return t.kind==="open-files"&&t.entity==="file"?t.revision_id??null:null}import{createHash as ui,randomUUID as ot}from"crypto";import{relative as di,resolve as an,sep as ci}from"path";function on(e){let t=process.env[e];return t==="1"||t==="true"||t==="yes"}function un(e,t){let n=e,r=new Set(n.safety?.network?.allowed_s3_buckets??[]);if(e.storage.type==="s3"&&e.storage.s3?.bucket)r.add(e.storage.s3.bucket);if(process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS)for(let i of process.env.HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.split(",").map((s)=>s.trim()).filter(Boolean))r.add(i);return{mode:e.mode,allowWriteRoots:[t.home,t.artifactsDir,t.cacheDir,t.exportsDir,t.indexesDir,t.logsDir,t.runsDir,t.schemasDir,t.wikiDir].map((i)=>an(i)),readOnlySourceAccess:!0,network:{webSearchEnabled:n.safety?.network?.web_search_enabled??on("HASNA_KNOWLEDGE_WEB_SEARCH"),s3ReadsEnabled:n.safety?.network?.s3_reads_enabled??on("HASNA_KNOWLEDGE_ALLOW_S3_READS"),allowedS3Buckets:[...r].sort()},redaction:{enabled:n.safety?.redaction?.enabled??!0},approvals:{generatedWritesRequireApproval:n.safety?.approvals?.generated_writes_require_approval??!0}}}function li(e,t){let n=di(e,t);return n===""||!n.startsWith("..")&&n!==".."&&!n.startsWith(`..${ci}`)}function Y(e,t){let n=an(e);if(!t.allowWriteRoots.some((r)=>li(r,n)))throw Error(`Safety policy denied write outside .hasna/apps/knowledge: ${e}`)}function B(e,t){let r=new URL(e).hostname;if(!t.network.s3ReadsEnabled)throw Error("Safety policy denied S3 read. Set safety.network.s3_reads_enabled=true or HASNA_KNOWLEDGE_ALLOW_S3_READS=1.");if(!t.network.allowedS3Buckets.includes(r))throw Error(`Safety policy denied S3 bucket "${r}". Add it to safety.network.allowed_s3_buckets or HASNA_KNOWLEDGE_ALLOWED_S3_BUCKETS.`)}function ee(e){if(!e.network.webSearchEnabled)throw Error("Safety policy denied web search. Set safety.network.web_search_enabled=true or HASNA_KNOWLEDGE_WEB_SEARCH=1.")}var _i=[{type:"private_key_block",severity:"high",regex:/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,replacement:"[REDACTED:private_key_block]"},{type:"secret_assignment",severity:"high",regex:/\b(?:api[_-]?key|secret|token|password)\s*[:=]\s*['"]?[^'"\s]{8,}/gi,replacement:"[REDACTED:secret_assignment]"},{type:"openai_api_key",severity:"high",regex:/\bsk-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:openai_api_key]"},{type:"anthropic_api_key",severity:"high",regex:/\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,replacement:"[REDACTED:anthropic_api_key]"},{type:"aws_access_key_id",severity:"high",regex:/\bA(?:KIA|SIA)[A-Z0-9]{16}\b/g,replacement:"[REDACTED:aws_access_key_id]"}];function Ne(e,t){if(t&&!t.redaction.enabled)return{text:e,findings:[]};let n=e,r=[];for(let i of _i)n=n.replace(i.regex,(s,...d)=>{let c=typeof d.at(-2)==="number"?d.at(-2):n.indexOf(s);return r.push({type:i.type,severity:i.severity,start:Math.max(0,c),end:Math.max(0,c+s.length)}),i.replacement});return{text:n,findings:r}}function fi(e){return`audit_${ui("sha256").update(`${e.event_type}\x00${e.action}\x00${e.target_uri??""}\x00${e.created_at??""}\x00${JSON.stringify(e.metadata??{})}\x00${ot()}`).digest("hex").slice(0,24)}`}function A(e,t){let n=t.created_at??new Date().toISOString(),r=fi({...t,created_at:n});return e.run(`INSERT INTO audit_events (id, event_type, action, target_uri, decision, metadata_json, created_at)
|
|
411
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[r,t.event_type,t.action,t.target_uri??null,t.decision,JSON.stringify(t.metadata??{}),n]),r}function Ae(e,t){let n=t.created_at??new Date().toISOString();for(let r of t.findings)e.run(`INSERT INTO redaction_findings (id, source_uri, run_id, severity, finding_type, metadata_json, created_at)
|
|
412
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,[`redact_${ot()}`,t.source_uri??null,t.run_id??null,r.severity,r.type,JSON.stringify({...t.metadata??{},start:r.start,end:r.end}),n]);return t.findings.length}function dn(e,t){let n=t.created_at??new Date().toISOString(),r=`approval_${ot()}`;return e.run(`INSERT INTO approval_gates (id, action, target_uri, status, reason, approved_by, metadata_json, created_at, updated_at)
|
|
413
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[r,t.action,t.target_uri??null,"approved",t.reason??null,t.approved_by??"local-cli",JSON.stringify(t.metadata??{}),n,n]),{id:r,status:"approved"}}function gi(e,t,n){let r=e.query(`SELECT id FROM approval_gates
|
|
412
414
|
WHERE action = ? AND status = 'approved' AND (target_uri IS NULL OR target_uri = ? OR ? IS NULL)
|
|
413
|
-
ORDER BY updated_at DESC LIMIT 1`).get(t,
|
|
415
|
+
ORDER BY updated_at DESC LIMIT 1`).get(t,n??null,n??null);return Boolean(r)}function cn(e,t,n,r){let i=n==="generated_write"&&t.approvals.generatedWritesRequireApproval,s=!i||gi(e,n,r);return{action:n,target_uri:r??null,approval_required:i,approved:s,decision:s?"allow":"requires_approval"}}function Ie(e,t){return`${e}_${pi("sha256").update(t).digest("hex").slice(0,20)}`}function te(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function R(e){return typeof e==="string"&&e.length>0?e:void 0}function ki(e){let t=R(e.source_ref)??R(e.source_uri)??R(e.uri);if(t)return t;let n=R(e.file_id);if(n){let s=R(e.revision_id)??R(e.revision),d=`open-files://file/${encodeURIComponent(n)}`;return s?`${d}/revision/${encodeURIComponent(s)}`:d}let r=R(e.source_id),i=R(e.path);if(r&&i)return`open-files://source/${encodeURIComponent(r)}/path/${encodeURIComponent(i)}`;throw Error("Outbox event is missing source_ref, file_id, or source_id/path.")}function bi(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function vi(e){return R(e.hash)??R(e.checksum)??R(e.sha256)??null}function wi(e,t,n){return R(e.revision_id)??R(e.revision)??R(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??n??null}function Ti(e){return(R(e.event)??R(e.type)??R(e.action)??R(e.change_type)??"changed").toLowerCase()}function Si(e){let t=R(e.path);return R(e.title)??R(e.name)??(t?yi(t):null)}function Ri(e,t){let n=ki(e),r=K(n),i=vi(e);return{raw:e,eventType:Ti(e),sourceRef:n,sourceUri:bi(n,r),kind:r.kind,title:Si(e),revision:wi(e,r,i),hash:i,status:R(e.status)?.toLowerCase()??null,updatedAt:R(e.updated_at)??t,acl:e.permissions??e.acl??void 0}}function xi(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let n=JSON.parse(t);if(!Array.isArray(n))throw Error("Outbox array parse failed.");return n.map((r)=>{let i=te(r);if(!i)throw Error("Outbox array entries must be objects.");return i})}if(t.startsWith("{"))try{let n=JSON.parse(t),r=te(n);if(!r)throw Error("Outbox object parse failed.");if(Array.isArray(r.events))return r.events.map((i)=>{let s=te(i);if(!s)throw Error("Outbox events entries must be objects.");return s});if("source_ref"in r||"source_uri"in r||"file_id"in r)return[r]}catch(n){let r=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(r.length<=1)throw n;return r.map((i)=>{let s=te(JSON.parse(i));if(!s)throw Error("Outbox JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((n)=>n.trim().length>0).map((n)=>{let r=te(JSON.parse(n));if(!r)throw Error("Outbox JSONL entries must be objects.");return r})}async function Oi(e,t,n){let r=new URL(e),i=r.hostname,s=decodeURIComponent(r.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 outbox URI: ${e}`);if(n)B(e,n);let[{S3Client:d,GetObjectCommand:c},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new d({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new c({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function Ni(e,t,n){if(e.startsWith("s3://"))return Oi(e,t,n);if(!hi(e))throw Error(`Outbox not found: ${e}`);return Ei(e,"utf8")}function ln(e,t){let n={};if(e)try{n=te(JSON.parse(e))??{}}catch{n={}}return JSON.stringify({...n,...t})}function Ai(e,t,n){let r=Ie("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
414
416
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
415
417
|
ON CONFLICT(uri) DO UPDATE SET
|
|
416
418
|
kind = excluded.kind,
|
|
417
419
|
title = COALESCE(excluded.title, sources.title),
|
|
418
|
-
updated_at = excluded.updated_at`,[
|
|
420
|
+
updated_at = excluded.updated_at`,[r,t.sourceUri,t.kind,t.title,JSON.stringify({source_ref:t.sourceRef,source_uri:t.sourceUri,status:t.status,last_outbox_event:t.eventType}),JSON.stringify(t.acl??{}),n,t.updatedAt]);let i=e.query("SELECT id, metadata_json, acl_json FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source for outbox event: ${t.sourceUri}`);let s={source_ref:t.sourceRef,source_uri:t.sourceUri,last_outbox_event:t.eventType,last_outbox_at:t.updatedAt};if(t.status)s.status=t.status;if(R(t.raw.path))s.path=t.raw.path;return e.run("UPDATE sources SET metadata_json = ?, acl_json = CASE WHEN ? IS NULL THEN acl_json ELSE ? END, updated_at = ? WHERE id = ?",[ln(i.metadata_json,s),t.acl===void 0?null:JSON.stringify(t.acl),t.acl===void 0?null:JSON.stringify(t.acl),t.updatedAt,i.id]),i.id}function Ii(e,t,n,r){if(!n.revision)return null;let i=Ie("rev",`${t}\x00${n.revision}`),s={source_ref:n.sourceRef,source_uri:n.sourceUri,status:n.status,last_outbox_event:n.eventType,reindex_required:!0};return e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
419
421
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
420
422
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
421
423
|
hash = COALESCE(excluded.hash, source_revisions.hash),
|
|
422
|
-
metadata_json = excluded.metadata_json`,[i,t,
|
|
423
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:
|
|
424
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[
|
|
425
|
-
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[
|
|
426
|
-
`);if(!
|
|
424
|
+
metadata_json = excluded.metadata_json`,[i,t,n.revision,n.hash,R(n.raw.extracted_text_ref)??null,JSON.stringify(s),r]),e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,n.revision)?.id??null}function Li(e,t,n){if(n.revision)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").all(t,n.revision).map((r)=>r.id);if(n.hash)return e.query("SELECT id FROM source_revisions WHERE source_id = ? AND hash = ?").all(t,n.hash).map((r)=>r.id);return e.query("SELECT id FROM source_revisions WHERE source_id = ?").all(t).map((r)=>r.id)}function Ci(e,t){let n=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t),r=0,i=0;for(let d of n){let c=e.query("SELECT COUNT(*) AS n FROM chunk_embeddings WHERE chunk_id = ?").get(d.id);r+=c?.n??0;let _=e.query("SELECT COUNT(*) AS n FROM vector_index_entries WHERE chunk_id = ?").get(d.id);i+=_?.n??0,e.run("DELETE FROM vector_index_entries WHERE chunk_id = ?",[d.id]),e.run("DELETE FROM chunk_embeddings WHERE chunk_id = ?",[d.id]),e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[d.id])}e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]);let s=e.query("SELECT metadata_json FROM source_revisions WHERE id = ?").get(t);return e.run("UPDATE source_revisions SET metadata_json = ? WHERE id = ?",[ln(s?.metadata_json,{reindex_required:!0,invalidated_at:new Date().toISOString()}),t]),{chunksDeleted:n.length,embeddingsDeleted:r,vectorEntriesDeleted:i}}function Pi(e,t){return t==="deleted"||["delete","deleted","remove","removed"].includes(e)}function Di(e){return["move","moved","rename","renamed","path_changed"].includes(e)}function Ui(e){return["permission","permissions","permission_changed","acl_changed"].includes(e)}async function _n(e){let t=(e.now??new Date).toISOString();if(e.safetyPolicy)Y(e.dbPath,e.safetyPolicy);S(e.dbPath);let n=await Ni(e.input,e.config,e.safetyPolicy),r=xi(n),i=b(e.dbPath),s=`run_${mi()}`;try{return i.transaction(()=>{i.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
425
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[s,"open-files-outbox",e.input,"completed","local","open-files-outbox",JSON.stringify({path:e.input,events:r.length}),t,t]);let d=new Set,c=new Set,_=0,a=0,u=0,o=0,l=0,f=0,g=0;return A(i,{event_type:"source_read",action:e.input.startsWith("s3://")?"s3_outbox_read":"local_outbox_read",target_uri:e.input,decision:"allow",metadata:{events:r.length,read_only:!0},created_at:t}),r.forEach((E,h)=>{let m=Ri(E,t),T=Ai(i,m,t);d.add(T);let k=Ii(i,T,m,t);if(k)c.add(k);let p=Li(i,T,m);for(let x of p){c.add(x);let N=Ci(i,x);_+=N.chunksDeleted,a+=N.embeddingsDeleted,u+=N.vectorEntriesDeleted,o+=1}if(Pi(m.eventType,m.status))l+=1;if(Di(m.eventType))f+=1;if(Ui(m.eventType)||m.acl!==void 0)g+=1;i.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
426
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[Ie("evt",`${s}\x00${h}\x00${m.sourceRef}\x00${m.eventType}`),s,"info",m.eventType,JSON.stringify({source_ref:m.sourceRef,source_uri:m.sourceUri,revision:m.revision,hash:m.hash,status:m.status,affected_revisions:p.length}),m.updatedAt])}),i.run(`INSERT INTO provider_usage (id, run_id, provider, model, input_tokens, output_tokens, cost_usd, metadata_json, created_at)
|
|
427
|
+
VALUES (?, ?, ?, ?, 0, 0, 0, ?, ?)`,[Ie("usage",s),s,"local","open-files-outbox",JSON.stringify({note:"No model provider used for outbox invalidation."}),t]),A(i,{event_type:"write",action:"knowledge_outbox_invalidation",target_uri:e.dbPath,decision:"allow",metadata:{run_id:s,events:r.length,sources:d.size,revisions:c.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:u},created_at:t}),{path:e.input,db_path:e.dbPath,run_id:s,events_seen:r.length,sources_touched:d.size,revisions_touched:c.size,chunks_deleted:_,embeddings_deleted:a,vector_entries_deleted:u,stale_revisions:o,deleted_sources:l,moved_sources:f,permission_updates:g}})()}finally{i.close()}}import{createHash as Ki}from"crypto";import{existsSync as Mi,readFileSync as Fi}from"fs";import{basename as ji}from"path";function at(e,t){return`${e}_${Ki("sha256").update(t).digest("hex").slice(0,20)}`}function ne(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:void 0}function w(e){return typeof e==="string"&&e.length>0?e:void 0}function Wi(e){return typeof e==="number"&&Number.isFinite(e)?e:void 0}function Xi(e){let t=w(e.source_ref)??w(e.source_uri)??w(e.uri);if(t)return t;let n=w(e.file_id);if(n){let s=w(e.revision_id)??w(e.revision),d=`open-files://file/${encodeURIComponent(n)}`;return s?`${d}/revision/${encodeURIComponent(s)}`:d}let r=w(e.source_id),i=w(e.path);if(r&&i)return`open-files://source/${encodeURIComponent(r)}/path/${encodeURIComponent(i)}`;throw Error("Manifest item is missing source_ref, file_id, or source_id/path.")}function $i(e,t){if(t.kind==="open-files"&&t.entity==="file"&&t.revision_id)return e.replace(/\/revision\/[^/]+$/,"");return e}function qi(e){let t=w(e.extracted_text)??w(e.text)??w(e.content_text)??w(e.markdown);if(t!==void 0)return t;let n=e.content;return typeof n==="string"?n:null}function Hi(e){let t=w(e.extracted_text_ref)??w(e.extracted_text_uri)??w(e.text_ref);if(t)return t;let n=ne(e.content);return w(n?.extracted_text_ref)??w(n?.extracted_text_uri)??null}function Bi(e){let t=w(e.path);return w(e.title)??w(e.name)??(t?ji(t):null)}function zi(e){return w(e.hash)??w(e.checksum)??w(e.sha256)??null}function Ji(e,t,n){return w(e.revision_id)??w(e.revision)??w(e.version_id)??(t.kind==="open-files"?t.revision_id:void 0)??n??w(e.updated_at)??"current"}function Gi(e,t){let n={};for(let[r,i]of Object.entries(e)){if(["text","content","content_text","extracted_text","markdown"].includes(r))continue;n[r]=i}return n.source_ref=t.sourceRef,n.source_uri=t.sourceUri,n.status=t.status,n}function Yi(e,t){let n=Xi(e),r=K(n),i=$i(n,r),s=zi(e),d=w(e.status)??"active";return{raw:e,sourceRef:n,sourceUri:i,kind:r.kind,title:Bi(e),revision:Ji(e,r,s),hash:s,extractedTextUri:Hi(e),text:qi(e),metadata:Gi(e,{sourceRef:n,sourceUri:i,status:d}),acl:e.permissions??e.acl??{},status:d,updatedAt:w(e.updated_at)??t}}function Vi(e){let t=e.trim();if(!t)return[];if(t.startsWith("[")){let n=JSON.parse(t);if(!Array.isArray(n))throw Error("Manifest array parse failed.");return n.map((r)=>{let i=ne(r);if(!i)throw Error("Manifest array entries must be objects.");return i})}if(t.startsWith("{"))try{let n=JSON.parse(t),r=ne(n);if(!r)throw Error("Manifest object parse failed.");if(Array.isArray(r.items))return r.items.map((i)=>{let s=ne(i);if(!s)throw Error("Manifest items entries must be objects.");return s});if("source_ref"in r||"source_uri"in r||"file_id"in r)return[r]}catch(n){let r=t.split(/\r?\n/).filter((i)=>i.trim().length>0);if(r.length<=1)throw n;return r.map((i)=>{let s=ne(JSON.parse(i));if(!s)throw Error("Manifest JSONL entries must be objects.");return s})}return t.split(/\r?\n/).filter((n)=>n.trim().length>0).map((n)=>{let r=ne(JSON.parse(n));if(!r)throw Error("Manifest JSONL entries must be objects.");return r})}async function Qi(e,t,n){let r=new URL(e),i=r.hostname,s=decodeURIComponent(r.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 manifest URI: ${e}`);if(n)B(e,n);let[{S3Client:d,GetObjectCommand:c},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new d({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new c({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function Zi(e,t,n){if(e.startsWith("s3://"))return Qi(e,t,n);if(!Mi(e))throw Error(`Manifest not found: ${e}`);return Fi(e,"utf8")}function es(e,t,n){let r=e.replace(/\r\n/g,`
|
|
428
|
+
`);if(!r.trim())return[];let i=[],s=0;while(s<r.length){let d=Math.min(r.length,s+t),c=d;if(d<r.length){let a=r.lastIndexOf(`
|
|
427
429
|
|
|
428
|
-
`,
|
|
430
|
+
`,d),u=r.lastIndexOf(". ",d),o=Math.max(a,u);if(o>s+Math.floor(t*0.5))c=o+(o===a?2:1)}let _=r.slice(s,c).trim();if(_)i.push({ordinal:i.length,text:_,startOffset:s,endOffset:c});if(c>=r.length)break;s=Math.max(0,c-n)}return i}function ts(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function ns(e,t){let n=e.query("SELECT id FROM chunks WHERE source_revision_id = ?").all(t);for(let r of n)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[r.id]);return e.run("DELETE FROM chunks WHERE source_revision_id = ?",[t]),n.length}function rs(e,t,n){let r=at("src",t.sourceUri);e.run(`INSERT INTO sources (id, uri, kind, title, metadata_json, acl_json, created_at, updated_at)
|
|
429
431
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
430
432
|
ON CONFLICT(uri) DO UPDATE SET
|
|
431
433
|
kind = excluded.kind,
|
|
432
434
|
title = excluded.title,
|
|
433
435
|
metadata_json = excluded.metadata_json,
|
|
434
436
|
acl_json = excluded.acl_json,
|
|
435
|
-
updated_at = excluded.updated_at`,[
|
|
437
|
+
updated_at = excluded.updated_at`,[r,t.sourceUri,t.kind,t.title,JSON.stringify(t.metadata),JSON.stringify(t.acl??{}),n,t.updatedAt]);let i=e.query("SELECT id FROM sources WHERE uri = ?").get(t.sourceUri);if(!i)throw Error(`Failed to upsert source: ${t.sourceUri}`);return i.id}function is(e,t,n,r){let i=at("rev",`${t}\x00${n.revision}`);e.run(`INSERT INTO source_revisions (id, source_id, revision, hash, extracted_text_uri, metadata_json, created_at)
|
|
436
438
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
437
439
|
ON CONFLICT(source_id, revision) DO UPDATE SET
|
|
438
440
|
hash = excluded.hash,
|
|
439
441
|
extracted_text_uri = excluded.extracted_text_uri,
|
|
440
|
-
metadata_json = excluded.metadata_json`,[i,t,
|
|
441
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
442
|
+
metadata_json = excluded.metadata_json`,[i,t,n.revision,n.hash,n.extractedTextUri,JSON.stringify(n.metadata),r]);let s=e.query("SELECT id FROM source_revisions WHERE source_id = ? AND revision = ?").get(t,n.revision);if(!s)throw Error(`Failed to upsert source revision: ${n.sourceRef}`);return s.id}function ss(e,t,n,r,i,s,d){if(!n.text||n.status.toLowerCase()==="deleted")return{chunksInserted:0,redactions:0};let c=Ne(n.text,d);if(c.findings.length>0)Ae(e,{source_uri:n.sourceUri,findings:c.findings,metadata:{source_ref:n.sourceRef,revision:n.revision},created_at:r}),A(e,{event_type:"redaction",action:"source_text_redact",target_uri:n.sourceUri,decision:"redacted",metadata:{findings:c.findings.length,source_ref:n.sourceRef,revision:n.revision},created_at:r});let _=es(c.text,i,s);for(let a of _){let u=at("chk",`${t}\x00${a.ordinal}\x00${a.text}`),o=H({source_ref:n.sourceRef,source_uri:n.sourceUri,source_kind:n.kind,source_revision_id:t,revision:n.revision,hash:n.hash,chunk_id:u,start_offset:a.startOffset,end_offset:a.endOffset,status:n.status,resolver:"open-files-read-only"}),l=Kt({source_ref:n.sourceRef,source_uri:n.sourceUri,source_kind:n.kind,source_revision_id:t,revision:n.revision,hash:n.hash,status:n.status,path:w(n.raw.path)??null,mime:w(n.raw.mime)??w(n.raw.content_type)??null,size:Wi(n.raw.size)??null},o);e.run(`INSERT INTO chunks (id, source_revision_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
443
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[u,t,"source",a.ordinal,a.text,ts(a.text),a.startOffset,a.endOffset,JSON.stringify(l),r]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[u,a.text,n.title??"",n.sourceUri])}return{chunksInserted:_.length,redactions:c.findings.length}}async function fn(e){let t=e.now??new Date;if(e.safetyPolicy)Y(e.dbPath,e.safetyPolicy);S(e.dbPath);let n=await Zi(e.input,e.config,e.safetyPolicy),r=Vi(n);return de({dbPath:e.dbPath,items:r,sourceLabel:e.input,safetyPolicy:e.safetyPolicy,now:t,maxChunkChars:e.maxChunkChars,chunkOverlapChars:e.chunkOverlapChars})}async function de(e){let t=(e.now??new Date).toISOString(),n=e.maxChunkChars??4000,r=e.chunkOverlapChars??200;if(n<500)throw Error("maxChunkChars must be at least 500.");if(r<0||r>=n)throw Error("chunkOverlapChars must be less than maxChunkChars.");if(e.safetyPolicy)Y(e.dbPath,e.safetyPolicy);S(e.dbPath);let i=b(e.dbPath);try{return i.transaction(()=>{let d=new Set,c=new Set,_=0,a=0,u=0,o=0;A(i,{event_type:"source_read",action:e.readAction??(e.sourceLabel.startsWith("s3://")?"s3_manifest_read":"local_manifest_read"),target_uri:e.sourceLabel,decision:"allow",metadata:{items:e.items.length,read_only:!0},created_at:t});for(let l of e.items){let f=Yi(l,t),g=rs(i,f,t),E=is(i,g,f,t);if(d.add(g),c.add(E),f.text||f.status.toLowerCase()==="deleted")a+=ns(i,E);let h=ss(i,E,f,t,n,r,e.safetyPolicy);_+=h.chunksInserted,u+=h.redactions}return A(i,{event_type:"write",action:"knowledge_manifest_ingest",target_uri:e.dbPath,decision:"allow",metadata:{items:e.items.length,sources:d.size,revisions:c.size,chunks_inserted:_,redactions:u},created_at:t}),{path:e.sourceLabel,db_path:e.dbPath,items_seen:e.items.length,sources_upserted:d.size,revisions_upserted:c.size,chunks_inserted:_,chunks_deleted:a,redactions:u,skipped:o}})()}finally{i.close()}}import{createHash as _s}from"crypto";import{existsSync as fs,readFileSync as gs}from"fs";import{basename as Pe}from"path";function Le(e){if(!e)return{};try{let t=JSON.parse(e);return t&&typeof t==="object"&&!Array.isArray(t)?t:{}}catch{return{}}}function V(e,t){for(let n of t){let r=e[n];if(typeof r==="string"&&r.length>0)return r}return null}function gn(e,t){for(let n of t){let r=e[n];if(typeof r==="number"&&Number.isFinite(r))return r}return null}function os(e,t){let n=e.mode;if(typeof n==="string"&&n!=="read_only")throw Error(`Source resolver denied ${t}. Permission mode is ${n}, expected read_only.`);let r=e.denied_purposes;if(Array.isArray(r)&&r.includes(t))throw Error(`Source resolver denied ${t}. Purpose is explicitly denied.`);let i=e.allowed_purposes;if(Array.isArray(i)&&i.length>0&&!i.includes(t))throw Error(`Source resolver denied ${t}. Allowed purposes: ${i.join(", ")}`)}function as(e,t,n){if(!t)return n;try{let r=K(e);if(r.kind==="open-files"&&r.entity==="file")return`${e}/revision/${encodeURIComponent(t.revision)}`}catch{return n}return n}function us(e,t,n){return e.query(`SELECT id, uri, kind, title, metadata_json, acl_json, updated_at
|
|
442
444
|
FROM sources
|
|
443
445
|
WHERE uri = ? OR uri = ?
|
|
444
446
|
ORDER BY CASE WHEN uri = ? THEN 0 ELSE 1 END
|
|
445
|
-
LIMIT 1`).get(t,
|
|
447
|
+
LIMIT 1`).get(t,n,t)??null}function ds(e,t,n){if(n)return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
446
448
|
FROM source_revisions
|
|
447
449
|
WHERE source_id = ? AND revision = ?
|
|
448
|
-
LIMIT 1`).get(t,
|
|
450
|
+
LIMIT 1`).get(t,n)??null;return e.query(`SELECT id, revision, hash, extracted_text_uri, metadata_json, created_at
|
|
449
451
|
FROM source_revisions
|
|
450
452
|
WHERE source_id = ?
|
|
451
453
|
ORDER BY created_at DESC, revision DESC
|
|
452
|
-
LIMIT 1`).get(t)??null}function
|
|
454
|
+
LIMIT 1`).get(t)??null}function cs(e,t){if(!t)return 0;return e.query("SELECT COUNT(*) AS n FROM chunks WHERE source_revision_id = ?").get(t)?.n??0}function ls(e,t,n){if(!t||n<=0)return[];return e.query(`SELECT id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json
|
|
453
455
|
FROM chunks
|
|
454
456
|
WHERE source_revision_id = ?
|
|
455
457
|
ORDER BY ordinal ASC
|
|
456
|
-
LIMIT ?`).all(t,
|
|
458
|
+
LIMIT ?`).all(t,n)}async function Ce(e){let t=e.purpose??"knowledge_answer",n=Math.max(0,Math.min(e.limit??10,100)),r=(e.now??new Date).toISOString(),i=K(e.sourceRef),s=rn(e.sourceRef,i),d=sn(e.sourceRef);if(e.safetyPolicy){if(!e.safetyPolicy.readOnlySourceAccess)throw Error("Safety policy denied source resolution.");Y(e.dbPath,e.safetyPolicy)}S(e.dbPath);let c=b(e.dbPath);try{return c.transaction(()=>{let _=us(c,s,e.sourceRef);if(!_)return A(c,{event_type:"source_read",action:"open_files_resolve_missing",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:s},created_at:r}),{source_ref:e.sourceRef,source_uri:s,purpose:t,read_only:!0,resolved:!1,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:null,revision:null,content:{mime:null,size:null,hash:null,text_available:!1,chunks_total:0,chunks_returned:0,char_count_returned:0,extracted_text_ref:null,bytes_available:!1,bytes_exposed:!1},chunks:[],citations:[]};let a=Le(_.metadata_json),u=Le(_.acl_json);try{os(u,t)}catch(p){throw A(c,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"deny",metadata:{purpose:t,read_only:!0,source_uri:_.uri,error:p instanceof Error?p.message:String(p)},created_at:r}),p}let o=ds(c,_.id,d),l=Le(o?.metadata_json),f=cs(c,o?.id??null),g=ls(c,o?.id??null,n),E=as(_.uri,o,e.sourceRef),h=g.map((p)=>{let x=Le(p.metadata_json),N={resolver:"open-files-read-only",mode:"local_catalog",purpose:t,read_only:!0,source_ref:V(x,["source_ref"])??E,source_uri:_.uri,source_revision_id:o?.id??null,revision:o?.revision??null,hash:o?.hash??V(x,["hash"]),chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,resolved_at:r},v=H({source_ref:N.source_ref,source_uri:N.source_uri,source_kind:_.kind,source_revision_id:N.source_revision_id,revision:N.revision,hash:N.hash,chunk_id:p.id,start_offset:p.start_offset,end_offset:p.end_offset,status:V(x,["status"]),resolver:N.resolver});return{id:p.id,kind:p.kind,ordinal:p.ordinal,text:p.text,token_count:p.token_count,start_offset:p.start_offset,end_offset:p.end_offset,metadata:x,evidence:N,provenance:v}}),m=h.map((p)=>({source_ref:p.evidence.source_ref,source_uri:_.uri,chunk_id:p.id,quote:p.text.slice(0,500),start_offset:p.start_offset,end_offset:p.end_offset,evidence:p.evidence,provenance:p.provenance}));A(c,{event_type:"source_read",action:"open_files_resolve",target_uri:e.sourceRef,decision:"allow",metadata:{purpose:t,read_only:!0,source_uri:_.uri,revision:o?.revision??null,chunks_returned:h.length,chunks_total:f},created_at:r});let T=V(a,["mime","content_type"])??V(l,["mime","content_type"]),k=gn(a,["size","size_bytes"])??gn(l,["size","size_bytes"]);return{source_ref:E,source_uri:_.uri,purpose:t,read_only:!0,resolved:!0,resolver:{name:"open-files-read-only",mode:"local_catalog",contract:"open-files-knowledge-source-v1"},source:{id:_.id,uri:_.uri,kind:_.kind,title:_.title,metadata:a,permissions:u,updated_at:_.updated_at},revision:o?{id:o.id,revision:o.revision,hash:o.hash,extracted_text_uri:o.extracted_text_uri,metadata:l,created_at:o.created_at,reindex_required:l.reindex_required===!0}:null,content:{mime:T,size:k,hash:o?.hash??V(a,["hash","checksum","sha256"]),text_available:f>0,chunks_total:f,chunks_returned:h.length,char_count_returned:h.reduce((p,x)=>p+x.text.length,0),extracted_text_ref:o?.extracted_text_uri??V(l,["extracted_text_ref","extracted_text_uri"]),bytes_available:!1,bytes_exposed:!1},chunks:h,citations:m}})()}finally{c.close()}}function re(e){return`sha256:${_s("sha256").update(e).digest("hex")}`}function ps(e){return e.replace(/<script[\s\S]*?<\/script>/gi," ").replace(/<style[\s\S]*?<\/style>/gi," ").replace(/<[^>]+>/g," ").replace(/ /g," ").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">").replace(/\s+\n/g,`
|
|
457
459
|
`).replace(/\n\s+/g,`
|
|
458
|
-
`).replace(/[ \t]{2,}/g," ").trim()}async function
|
|
460
|
+
`).replace(/[ \t]{2,}/g," ").trim()}async function ms(e,t,n){let r=new URL(e),i=r.hostname,s=decodeURIComponent(r.pathname.replace(/^\/+/,""));if(!i||!s)throw Error(`Invalid S3 source URI: ${e}`);if(n)B(e,n);let[{S3Client:d,GetObjectCommand:c},{fromIni:_}]=await Promise.all([import("@aws-sdk/client-s3"),import("@aws-sdk/credential-providers")]),a=t?.storage.type==="s3"&&t.storage.s3?.bucket===i?t.storage.s3:void 0,o=await new d({region:a?.region,credentials:a?.profile?_({profile:a.profile}):void 0,maxAttempts:a?.max_attempts}).send(new c({Bucket:i,Key:s}));if(!o.Body)return"";return await o.Body.transformToString()}async function hs(e,t){if(t)ee(t);let n=await fetch(e,{headers:{accept:"text/markdown,text/plain,text/html,application/json;q=0.8,*/*;q=0.5","user-agent":"@hasna/knowledge source-ingest"}});if(!n.ok)throw Error(`Web source read failed ${n.status}: ${e}`);let r=n.headers.get("content-type"),i=await n.text();return{text:r?.includes("html")?ps(i):i,mime:r}}function De(e){if(e.kind==="file")return Pe(e.path);if(e.kind==="s3")return Pe(e.key);if(e.kind==="web")return Pe(new URL(e.url).pathname)||e.url;return e.path?Pe(e.path):e.id}async function pn(e,t,n){if(e.kind==="file"){if(!fs(e.path))throw Error(`Source file not found: ${e.path}`);let r=gs(e.path,"utf8");return{text:r,contentSource:"file",title:De(e),mime:"text/plain",size:r.length,hash:re(r),revision:null,extractedTextRef:null,metadata:{path:e.path},permissions:{mode:"read_only"}}}if(e.kind==="s3"){let r=await ms(e.uri,t,n);return{text:r,contentSource:"s3",title:De(e),mime:"text/plain",size:r.length,hash:re(r),revision:null,extractedTextRef:null,metadata:{bucket:e.bucket,key:e.key},permissions:{mode:"read_only"}}}if(e.kind==="web"){let r=await hs(e.url,n);return{text:r.text,contentSource:"web",title:De(e),mime:r.mime,size:r.text.length,hash:re(r.text),revision:null,extractedTextRef:null,metadata:{url:e.url},permissions:{mode:"read_only"}}}throw Error(`Direct source reading is not available for ${e.uri}`)}async function Es(e,t,n){if(e.startsWith("open-files://"))throw Error("Open-files extracted text refs require an open-files resolver API. Ingest an open-files manifest with extracted_text or an extracted_text_ref using file://, s3://, or https://.");let r=K(e);return{text:(await pn(r,t,n)).text,contentSource:"extracted_text_ref"}}async function ys(e){let t=await Ce({dbPath:e.dbPath,sourceRef:e.sourceRef,purpose:e.purpose??"knowledge_index",limit:100,safetyPolicy:e.safetyPolicy,now:e.now});if(!t.resolved)throw Error("Open-files source is not in the local knowledge catalog. Ingest an open-files manifest first or use the open-files resolver API.");if(t.revision?.extracted_text_uri&&!t.content.text_available){let r=await Es(t.revision.extracted_text_uri,e.config,e.safetyPolicy);return{text:r.text,contentSource:r.contentSource,title:t.source?.title??null,mime:t.content.mime,size:r.text.length,hash:t.revision.hash??re(r.text),revision:t.revision.revision,extractedTextRef:t.revision.extracted_text_uri,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}if(t.chunks.length===0)throw Error("Open-files source has no extracted text chunks yet. Ingest an open-files manifest with extracted_text or extracted_text_ref first.");let n=t.chunks.map((r)=>r.text).join(`
|
|
459
461
|
|
|
460
|
-
`);return{text:
|
|
462
|
+
`);return{text:n,contentSource:"catalog_chunks",title:t.source?.title??null,mime:t.content.mime,size:n.length,hash:t.revision?.hash??re(n),revision:t.revision?.revision??null,extractedTextRef:t.revision?.extracted_text_uri??null,metadata:t.source?.metadata??{},permissions:t.source?.permissions??{mode:"read_only"}}}function ks(e,t,n,r){let i=n.hash??re(n.text),s={...n.metadata,source_ref:e,content_source:n.contentSource,read_only:!0},d={source_ref:e,name:n.title??De(t),mime:n.mime??"text/plain",size:n.size??n.text.length,hash:i,revision:n.revision??i,status:"active",updated_at:new Date().toISOString(),permissions:{mode:"read_only",allowed_purposes:[r],...n.permissions},metadata:s,extracted_text_ref:n.extractedTextRef,extracted_text:n.text};if(t.kind==="open-files"){if(t.entity==="file")d.file_id=t.id;if(t.entity==="source")d.source_id=t.id,d.path=t.path}if(t.kind==="file")d.path=t.path;if(t.kind==="s3")d.path=t.key;if(t.kind==="web")d.url=t.url;return d}async function mn(e){let t=e.purpose??"knowledge_index",n=K(e.sourceRef),r=n.kind==="open-files"?await ys(e):await pn(n,e.config,e.safetyPolicy),i=ks(e.sourceRef,n,r,t);return{...await de({dbPath:e.dbPath,items:[i],sourceLabel:e.sourceRef,readAction:"source_ref_ingest_read",safetyPolicy:e.safetyPolicy,now:e.now}),source_ref:e.sourceRef,content_source:r.contentSource,read_only:!0,hash:String(i.hash)}}import{createHash as bs,randomUUID as hn}from"crypto";function vs(e,t){return`${e}_${bs("sha256").update(t).digest("hex").slice(0,20)}`}function ws(e){let t=b(e);try{let n=t.query("SELECT status, COUNT(*) AS n FROM reindex_queue GROUP BY status ORDER BY status").all();return Object.fromEntries(n.map((r)=>[r.status,r.n]))}finally{t.close()}}function En(e,t){let n=G(t.modelRef,t.config),r=L(n),i=b(e);try{return i.query(`SELECT c.id AS chunk_id, c.source_revision_id, s.uri AS source_uri
|
|
461
463
|
FROM chunks c
|
|
462
464
|
LEFT JOIN source_revisions sr ON sr.id = c.source_revision_id
|
|
463
465
|
LEFT JOIN sources s ON s.id = sr.source_id
|
|
464
466
|
LEFT JOIN vector_index_entries v ON v.chunk_id = c.id AND v.provider = ? AND v.model = ?
|
|
465
467
|
WHERE v.id IS NULL
|
|
466
|
-
ORDER BY c.created_at ASC, c.ordinal ASC`).all(
|
|
467
|
-
WHERE metadata_json LIKE '%"reindex_required":true%' OR metadata_json LIKE '%"status":"stale"%'`).get()?.n??0;return{schema_version:
|
|
468
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[a,"embedding",_.chunk_id,_.source_uri,
|
|
468
|
+
ORDER BY c.created_at ASC, c.ordinal ASC`).all(r.provider,r.model)}finally{i.close()}}function yn(e){S(e.dbPath);let t=b(e.dbPath);try{let n=t.query("SELECT MAX(version) AS version FROM schema_versions").get()?.version??0,r=t.query("SELECT COUNT(*) AS n FROM chunks").get()?.n??0,i=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0,s=En(e.dbPath,e).length,d=t.query(`SELECT COUNT(*) AS n FROM source_revisions
|
|
469
|
+
WHERE metadata_json LIKE '%"reindex_required":true%' OR metadata_json LIKE '%"status":"stale"%'`).get()?.n??0;return{schema_version:n,chunks:r,vector_entries:i,missing_embeddings:s,queued:ws(e.dbPath),stale_revisions:d}}finally{t.close()}}function ut(e){S(e.dbPath);let t=(e.now??new Date).toISOString(),n=e.reason??"missing_embedding",r=En(e.dbPath,e),i=b(e.dbPath),s=0,d=0;try{i.transaction(()=>{for(let _ of r){let a=vs("rq",`embedding\x00${_.chunk_id}\x00${n}`);if(i.query("SELECT id FROM reindex_queue WHERE kind = ? AND target_id = ? AND reason = ?").get("embedding",_.chunk_id,n)){d+=1;continue}i.run(`INSERT INTO reindex_queue (id, kind, target_id, source_uri, reason, status, metadata_json, created_at, updated_at)
|
|
470
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[a,"embedding",_.chunk_id,_.source_uri,n,"pending",JSON.stringify({source_revision_id:_.source_revision_id}),t,t]),s+=1}})()}finally{i.close()}return{enqueued:s,already_queued:d,reason:n}}function Ts(e){let t=b(e);try{let n=t.query("SELECT COUNT(*) AS n FROM chunk_embeddings").get()?.n??0,r=t.query("SELECT COUNT(*) AS n FROM vector_index_entries").get()?.n??0;return t.run("DELETE FROM vector_index_entries"),t.run("DELETE FROM chunk_embeddings"),{embeddings:n,vectorEntries:r}}finally{t.close()}}function Ss(e,t,n){let r=G(t.modelRef,t.config),i=L(r),s=b(e);try{return s.run(`UPDATE reindex_queue
|
|
469
471
|
SET status = ?, updated_at = ?
|
|
470
472
|
WHERE kind = ?
|
|
471
473
|
AND status = ?
|
|
@@ -474,12 +476,12 @@ ${l}`;else{let{generateText:p}=await import("ai"),R=await kt(i,{config:e.config,
|
|
|
474
476
|
WHERE v.chunk_id = reindex_queue.target_id
|
|
475
477
|
AND v.provider = ?
|
|
476
478
|
AND v.model = ?
|
|
477
|
-
)`,["completed",
|
|
478
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
479
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${
|
|
480
|
-
`),
|
|
481
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[o,"provider-web-search",t,"running",a,
|
|
482
|
-
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${
|
|
479
|
+
)`,["completed",n,"embedding","pending",i.provider,i.model]).changes}finally{s.close()}}async function kn(e){S(e.dbPath);let t=(e.now??new Date).toISOString(),n=`run_${hn()}`,r=e.full?Ts(e.dbPath):{embeddings:0,vectorEntries:0},i=ut({...e,reason:e.full?"full_embedding_rebuild":"missing_embedding"}),s=b(e.dbPath);try{s.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
480
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[n,"embedding-refresh",e.full?"full":"incremental","running","local",G(e.modelRef,e.config),JSON.stringify({full:e.full===!0,queued:i}),t,t])}finally{s.close()}let d=await be({dbPath:e.dbPath,config:e.config,env:e.env,modelRef:e.modelRef,dimensions:e.dimensions,fake:e.fake,limit:e.limit,now:e.now}),c=Ss(e.dbPath,e,t),_=b(e.dbPath);try{_.run("UPDATE runs SET status = ?, metadata_json = ?, updated_at = ? WHERE id = ?",["completed",JSON.stringify({full:e.full===!0,queued:i,indexed:d,completed_queue_items:c}),t,n]),_.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
481
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${hn()}`,n,"info","embedding_refresh_completed",JSON.stringify({queued:i,indexed:d,completed_queue_items:c}),t])}finally{_.close()}return{run_id:n,full:e.full===!0,deleted_embeddings:r.embeddings,deleted_vector_entries:r.vectorEntries,queued:i,indexed:d,completed_queue_items:c}}var Ue=1;function bn(e){return Boolean(e&&typeof e==="object"&&!Array.isArray(e))}function F(e,t){let n=e[t];return typeof n==="string"?n:void 0}function Rs(e,t){let n=e[t];return typeof n==="number"&&Number.isFinite(n)?n:void 0}function vn(e,t){let n=e[t];return Array.isArray(n)?n:void 0}function ce(e,t){let n=bn(e)?e:{};return{contract_version:Ue,id:F(n,"id")??t?.id,type:F(n,"type")??t?.type,status:F(n,"status")??t?.status,query:F(n,"query")??t?.query,prompt:F(n,"prompt")??t?.prompt,output_preview:Object.prototype.hasOwnProperty.call(n,"output_preview")?n.output_preview:t?.output_preview,citations:vn(n,"citations")??t?.citations,artifacts:vn(n,"artifacts")??t?.artifacts,usage:bn(n.usage)?n.usage:t?.usage,created_at:F(n,"created_at")??t?.created_at,started_at:F(n,"started_at")??t?.started_at,completed_at:F(n,"completed_at")??t?.completed_at,duration_ms:Rs(n,"duration_ms")??t?.duration_ms,error_code:F(n,"error_code")??t?.error_code,error_message:F(n,"error_message")??t?.error_message,error:F(n,"error")??t?.error,details:Object.prototype.hasOwnProperty.call(n,"details")?n.details:t?.details}}function wn(e){return{contract_version:Ue,service:"open-knowledge",mode:e.mode,capabilities:["registry","search","ask","build","sync","status","logs","artifacts","open-files-source-refs","s3-generated-artifacts"],endpoints:{registry:"/api/v1/knowledge/registry",search:"/api/v1/knowledge/search",ask:"/api/v1/knowledge/ask",build:"/api/v1/knowledge/build",sync:"/api/v1/knowledge/sync",run_status:"/api/v1/knowledge/runs/{run_id}",run_logs:"/api/v1/knowledge/runs/{run_id}/logs",run_artifacts:"/api/v1/knowledge/runs/{run_id}/artifacts"},source_contract:{owner:"open-files",preferred_ref:"open-files",allowed_schemes:e.sourceSchemes,raw_source_bytes_stored_in_open_knowledge:!1},artifact_contract:{storage_type:e.storageType,uri_prefix:e.artifactUriPrefix,generated_only:!0}}}class Ke{apiKey;apiUrl;constructor(e,t){this.apiKey=e;this.apiUrl=t}static fromConfig(e,t=process.env){let n=Ge(t);if(!n.apiKey)return null;return new Ke(n.apiKey,me(e,t))}async request(e,t={}){return fetch(`${this.apiUrl}${e}`,{...t,headers:{Authorization:`Bearer ${this.apiKey}`,"Content-Type":"application/json",...t.headers}})}async registry(){return(await this.request("/api/v1/knowledge/registry")).json()}async search(e){let t=await this.request("/api/v1/knowledge/search",{method:"POST",body:JSON.stringify(e)});return ce(await t.json(),{type:"search",query:e.query})}async ask(e){let t=await this.request("/api/v1/knowledge/ask",{method:"POST",body:JSON.stringify(e)});return ce(await t.json(),{type:"ask",prompt:e.prompt})}async build(e){let t=await this.request("/api/v1/knowledge/build",{method:"POST",body:JSON.stringify(e)});return ce(await t.json(),{type:"build",prompt:e.prompt})}async sync(e={}){let t=await this.request("/api/v1/knowledge/sync",{method:"POST",body:JSON.stringify(e)});return ce(await t.json(),{type:"sync"})}async runStatus(e){let t=await this.request(`/api/v1/knowledge/runs/${encodeURIComponent(e)}`);if(!t.ok)return null;return ce(await t.json(),{id:e,type:"status"})}async runLogs(e){let t=await this.request(`/api/v1/knowledge/runs/${encodeURIComponent(e)}/logs`);if(!t.ok)return[];let n=await t.json();return Array.isArray(n)?n:[]}async runArtifacts(e){let t=await this.request(`/api/v1/knowledge/runs/${encodeURIComponent(e)}/artifacts`);if(!t.ok)return[];let n=await t.json();return Array.isArray(n)?n:[]}}import{createHash as xs,randomUUID as Tn}from"crypto";function Os(e){return`sha256:${xs("sha256").update(e).digest("hex")}`}function Sn(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Rn(e){return e&&typeof e==="object"&&!Array.isArray(e)?e:{}}function z(e){return typeof e==="string"&&e.length>0?e:null}function Ns(e){let t=Rn(e),n=z(t.url)??z(t.uri)??z(t.sourceUrl);if(!n)return null;return{url:n,title:z(t.title)??z(t.name),snippet:z(t.snippet)??z(t.text)??z(t.description),provider_metadata:t}}function Me(e,t){if(Array.isArray(e)){for(let i of e)Me(i,t);return}let n=Ns(e);if(n)t.set(n.url,n);let r=Rn(e);for(let i of["sources","results","citations","annotations","output"])if(r[i])Me(r[i],t)}function As(e,t){return Array.from({length:Math.min(t,3)},(n,r)=>({url:`https://example.com/knowledge-web-${r+1}`,title:`Fake web source ${r+1}`,snippet:`Deterministic web-search fixture for "${e}"`,provider_metadata:{fake:!0,rank:r+1}}))}async function Is(e){let{generateText:t}=await import("ai"),{createOpenAI:n}=await import("@ai-sdk/openai"),r=$(e.config,"openai"),i=n({apiKey:e.env[r.api_key_env],baseURL:r.base_url}),s=i.tools?.webSearch;if(!s)throw Error("OpenAI provider does not expose tools.webSearch.");return t({model:i(e.model),prompt:e.query,tools:{web_search:s({externalWebAccess:!0,searchContextSize:"medium",...e.domains.length>0?{allowedDomains:e.domains}:{}})},toolChoice:{type:"tool",toolName:"web_search"}})}async function Ls(e){let{generateText:t}=await import("ai"),{createAnthropic:n}=await import("@ai-sdk/anthropic"),r=$(e.config,"anthropic"),i=n({apiKey:e.env[r.api_key_env],baseURL:r.base_url}),s=i.tools?.webSearch_20250305??i.tools?.webSearch;if(!s)throw Error("Anthropic provider does not expose a web search tool.");return t({model:i(e.model),prompt:e.query,tools:{web_search:s({maxUses:e.maxUses,...e.domains.length>0?{allowedDomains:e.domains}:{}})}})}async function Cs(e,t,n){if(!e.fileResults||t.length===0)return 0;let r=t.map((s)=>{let d=[s.title,s.snippet,s.url].filter(Boolean).join(`
|
|
482
|
+
`),c=Os(d);return{source_ref:s.url,name:s.title??s.url,url:s.url,mime:"text/plain",hash:c,revision:c,status:"active",updated_at:n,permissions:{mode:"read_only",allowed_purposes:["knowledge_answer","knowledge_index"]},metadata:{source_ref:s.url,content_source:"provider_web_search",provider_metadata:s.provider_metadata},extracted_text:d}});return(await de({dbPath:e.dbPath,items:r,sourceLabel:`web-search:${e.query}`,readAction:"provider_web_search_file_results",safetyPolicy:e.safetyPolicy,now:new Date(n)})).sources_upserted}async function xn(e){let t=e.query.trim();if(!t)throw Error("Web search query is required.");let n=e.env??process.env,r=(e.now??new Date).toISOString(),i=Math.max(1,Math.min(e.limit??5,20)),s=Math.max(1,Math.min(e.maxUses??3,10)),d=e.domains??[],c=q(e.modelRef??(e.provider?`${e.provider}:${$(e.config,e.provider).default_model}`:"default"),e.config),_=L(c),a=e.provider??_.provider,u=_.provider===a?_.model:$(e.config,a).default_model,o=`run_${Tn()}`;if(!e.fake&&e.safetyPolicy)ee(e.safetyPolicy);if(!e.fake&&a!=="openai"&&a!=="anthropic")throw Error(`Provider ${a} does not expose native web search yet.`);if(!e.fake)J(a,e.config,n);S(e.dbPath);let l=b(e.dbPath);try{l.run(`INSERT INTO runs (id, type, prompt, status, provider, model, metadata_json, created_at, updated_at)
|
|
483
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,[o,"provider-web-search",t,"running",a,u,JSON.stringify({domains:d,max_uses:s,fake:e.fake===!0}),r,r]),A(l,{event_type:"source_read",action:e.fake?"fake_provider_web_search":"provider_web_search",target_uri:t,decision:"allow",metadata:{provider:a,model:u,domains:d,max_uses:s},created_at:r})}finally{l.close()}let f="",g=[],E={input_tokens:Sn(t),output_tokens:0,cost_usd:0},h=[];if(e.fake)g=As(t,i),f=`Fake web search answer for: ${t}`,E.output_tokens=Sn(f);else{let k=a==="openai"?await Is({query:t,model:u,config:e.config,env:n,maxUses:s,domains:d}):await Ls({query:t,model:u,config:e.config,env:n,maxUses:s,domains:d});f=k.text;let p=new Map;Me(k.sources,p),Me(k.toolResults,p),g=Array.from(p.values()).slice(0,i);let x=Ee({provider:a,model:u,usage:k.usage,providerMetadata:k.providerMetadata});E={input_tokens:x.input_tokens,output_tokens:x.output_tokens,cost_usd:x.cost_usd}}let m=await Cs(e,g,r),T=b(e.dbPath);try{T.run("UPDATE runs SET status = ?, metadata_json = ?, updated_at = ? WHERE id = ?",["completed",JSON.stringify({domains:d,max_uses:s,sources:g.length,filed_sources:m,fake:e.fake===!0}),r,o]),T.run(`INSERT INTO run_events (id, run_id, level, event, metadata_json, created_at)
|
|
484
|
+
VALUES (?, ?, ?, ?, ?, ?)`,[`evt_${Tn()}`,o,"info","provider_web_search_completed",JSON.stringify({sources:g.length,filed_sources:m}),r]),ye(T,{run_id:o,provider:a,model:u,input_tokens:E.input_tokens,output_tokens:E.output_tokens,cost_usd:E.cost_usd,metadata:{web_search:!0,sources:g.length,filed_sources:m},created_at:r})}finally{T.close()}if(g.length===0)h.push("no_web_sources_returned");return{run_id:o,query:t,provider:a,model:u,answer:f,sources:g,filed_sources:m,usage:E,warnings:h}}import{createHash as Ps,randomUUID as Ds}from"crypto";var On=[{kind:"schema",prefix:"schemas/",description:"Machine-readable agent schemas and source rules."},{kind:"index",prefix:"indexes/",description:"Small orientation indexes and future shard manifests."},{kind:"log",prefix:"logs/",description:"Append-only JSONL run and wiki-maintenance log partitions."},{kind:"run",prefix:"runs/",description:"Prompt/tool/cost ledgers and generated output records."},{kind:"wiki_page",prefix:"wiki/",description:"Generated cited Markdown pages, not raw source files."},{kind:"export",prefix:"exports/",description:"Portable exports and snapshots of derived knowledge state."}];function Nn(e){let t=typeof e==="string"?Buffer.from(e):Buffer.from(e);return{hash:`sha256:${Ps("sha256").update(t).digest("hex")}`,size_bytes:t.byteLength}}function An(e){return On.find((n)=>e.startsWith(n.prefix))?.kind??"artifact"}function In(e,t,n="global"){let r=dt(e,t),i=e.storage.s3??null,s=i?.prefix?.replace(/^\/+|\/+$/g,"")??"",d=i?`s3://${i.bucket}/${s?`${s}/`:""}`:"";return{scope:n,mode:e.mode,storage_type:e.storage.type,workspace_home:t.home,local_layout:{app_path:se,config_path:t.configPath,json_store_path:t.jsonStorePath,knowledge_db_path:t.knowledgeDbPath,directories:{artifacts:t.artifactsDir,cache:t.cacheDir,exports:t.exportsDir,indexes:t.indexesDir,logs:t.logsDir,runs:t.runsDir,schemas:t.schemasDir,wiki:t.wikiDir}},artifact_store:{type:e.storage.type,artifacts_root:e.storage.artifacts_root,uri_prefix:e.storage.type==="s3"?d:`file://${t.artifactsDir}/`,s3:i?{bucket:i.bucket,prefix:s,region:i.region??null,profile:i.profile??null,server_side_encryption:i.server_side_encryption??null,kms_key_configured:Boolean(i.kms_key_id)}:null},hosted:{enabled:e.mode==="hosted",api_url:X(e.hosted?.api_url??Je),api_url_env:"KNOWLEDGE_API_URL",api_key_env:"KNOWLEDGE_API_KEY",auth_storage:"~/.hasna/knowledge/auth.json",remote_contract_version:Ue,requires_hosted_account_for_local_use:!1},source_ownership:{owner:"open-files",preferred_ref:e.sources.preferred_ref,allowed_schemes:e.sources.allowed_schemes,raw_source_bytes_stored_in_open_knowledge:!1,stores:["source refs","source revisions and hashes","citation spans","redacted extracted chunks","embeddings","generated wiki artifacts","indexes","run ledgers"],does_not_store:["raw open-files bytes","S3 object credentials","connector secrets","hosted tenant ownership state"]},generated_artifacts:On,scalability:{catalog:"knowledge.db tracks sources, revisions, chunks, citations, indexes, runs, and storage_objects.",indexes:"Indexes are cataloged DB rows plus sharded artifacts, not one giant index.md.",logs:"Logs use dated JSONL partitions under logs/yyyy/mm/dd.jsonl.",markdown:"Markdown pages are the readable wiki layer over DB/object-store state."},warnings:r.warnings}}function dt(e,t){let n=[],r=[];if(!t.home.endsWith(se))r.push(`Workspace home does not end with ${se}: ${t.home}`);if(e.storage.type==="s3"){if(!e.storage.s3?.bucket)n.push("storage.s3.bucket is required when storage.type is s3.");if(!e.storage.s3?.prefix)r.push("storage.s3.prefix is empty; generated knowledge artifacts will be written at the bucket root.");if(e.mode==="local")r.push("storage.type is s3 while mode is local; this is valid for BYO S3, but hosted wrappers should set mode to hosted.")}if(e.storage.type==="local"&&e.storage.s3)r.push("storage.s3 is configured but ignored while storage.type is local.");if(e.sources.preferred_ref!=="open-files")r.push("sources.preferred_ref should stay open-files for durable company knowledge.");if(!e.sources.allowed_schemes.includes("open-files"))n.push("sources.allowed_schemes must include open-files.");if(e.mode==="hosted"&&e.hosted?.api_url)try{X(e.hosted.api_url)}catch{n.push("hosted.api_url must be an http(s) URL when mode is hosted.")}return{ok:n.length===0,errors:n,warnings:r}}function Ln(e,t,n=new Date){let r=n.toISOString(),i=e.prepare(`
|
|
483
485
|
INSERT INTO storage_objects (
|
|
484
486
|
id, artifact_uri, kind, content_type, hash, size_bytes, metadata_json, created_at, updated_at
|
|
485
487
|
)
|
|
@@ -491,7 +493,7 @@ ${l}`;else{let{generateText:p}=await import("ai"),R=await kt(i,{config:e.config,
|
|
|
491
493
|
size_bytes = excluded.size_bytes,
|
|
492
494
|
metadata_json = excluded.metadata_json,
|
|
493
495
|
updated_at = excluded.updated_at
|
|
494
|
-
`);e.transaction((
|
|
496
|
+
`);e.transaction((d)=>{for(let c of d)i.run(Ds(),c.uri,c.kind,c.content_type??null,c.hash??null,c.size_bytes??null,JSON.stringify({key:c.key,...c.metadata??{}}),r,r)})(t)}import{createHash as Us}from"crypto";function Ks(e){let t=String(e.getUTCFullYear()),n=String(e.getUTCMonth()+1).padStart(2,"0"),r=String(e.getUTCDate()).padStart(2,"0");return{year:t,month:n,day:r}}function ct(e,t){return`${e}_${Us("sha256").update(t).digest("hex").slice(0,20)}`}function Ms(e){let t=e.trim().split(/\s+/).filter(Boolean).length;return Math.max(1,Math.ceil(t*1.25))}function Fs(){return`# Knowledge Agent Schema v1
|
|
495
497
|
|
|
496
498
|
## Source Rules
|
|
497
499
|
|
|
@@ -516,7 +518,7 @@ ${l}`;else{let{generateText:p}=await import("ai"),R=await kt(i,{config:e.config,
|
|
|
516
518
|
## Lint Rules
|
|
517
519
|
|
|
518
520
|
- Flag stale pages, missing citations, contradictions, orphan pages, duplicate pages, and unresolved source refs.
|
|
519
|
-
`}function
|
|
521
|
+
`}function js(){return`# Knowledge Index
|
|
520
522
|
|
|
521
523
|
This is a compact orientation index for agents. It is not the full search index.
|
|
522
524
|
|
|
@@ -531,19 +533,19 @@ This is a compact orientation index for agents. It is not the full search index.
|
|
|
531
533
|
|
|
532
534
|
Raw source files are resolved through open-files. This app stores source refs,
|
|
533
535
|
citations, chunks, generated wiki artifacts, indexes, and run records.
|
|
534
|
-
`}function
|
|
536
|
+
`}function Cn(){return`# Wiki
|
|
535
537
|
|
|
536
538
|
Generated durable knowledge pages live here.
|
|
537
539
|
|
|
538
540
|
Pages should be concise, cited, and organized for both humans and agents.
|
|
539
|
-
`}async function
|
|
540
|
-
`,content_type:"application/x-ndjson"}],o=await Promise.all(
|
|
541
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[
|
|
541
|
+
`}async function Pn(e,t=new Date){let{year:n,month:r,day:i}=Ks(t),s="schemas/v1.md",d="indexes/root.md",c="wiki/README.md",_=`logs/${n}/${r}/${i}.jsonl`,a={ts:t.toISOString(),event:"wiki_layout_initialized",schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md"},u=[{key:"schemas/v1.md",body:Fs(),content_type:"text/markdown"},{key:"indexes/root.md",body:js(),content_type:"text/markdown"},{key:"wiki/README.md",body:Cn(),content_type:"text/markdown"},{key:_,body:`${JSON.stringify(a)}
|
|
542
|
+
`,content_type:"application/x-ndjson"}],o=await Promise.all(u.map(async(l)=>{let f=await e.put(l);return{key:f.key,uri:f.uri,kind:An(l.key),content_type:l.content_type,metadata:{provenance:Ze({generated_from:"wiki_layout_init",artifact_key:l.key,citation_required:l.key.startsWith("wiki/")||l.key.startsWith("indexes/")})},...Nn(l.body)}}));return{schema_key:"schemas/v1.md",root_index_key:"indexes/root.md",wiki_readme_key:"wiki/README.md",log_key:_,artifacts:o,written:["schemas/v1.md","indexes/root.md","wiki/README.md",_]}}function lt(e){let t=e.metadata?.provenance;if(t&&typeof t==="object"&&!Array.isArray(t))return t;return Ze({generated_from:"wiki_layout_init",artifact_key:e.key})}function Ws(e,t,n,r,i,s){let d=lt(r),c=ct("chk",`${t}\x00${r.hash??r.uri}`),_=e.query("SELECT id FROM chunks WHERE wiki_page_id = ?").all(t);for(let a of _)e.run("DELETE FROM chunks_fts WHERE chunk_id = ?",[a.id]);e.run("DELETE FROM chunks WHERE wiki_page_id = ?",[t]),e.run(`INSERT INTO chunks (id, wiki_page_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
543
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,[c,t,"wiki",0,i,Ms(i),0,i.length,JSON.stringify({artifact_key:r.key,artifact_uri:r.uri,content_hash:r.hash??null,provenance:d}),s]),e.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)",[c,i,n,r.uri])}function Dn(e,t,n=new Date){let r=n.toISOString(),i=t.find((d)=>d.key.endsWith("indexes/root.md")),s=t.find((d)=>d.key.endsWith("wiki/README.md"));if(i)e.run(`INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
|
|
542
544
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
543
545
|
ON CONFLICT(kind, name, shard_key) DO UPDATE SET
|
|
544
546
|
artifact_uri = excluded.artifact_uri,
|
|
545
547
|
metadata_json = excluded.metadata_json,
|
|
546
|
-
updated_at = excluded.updated_at`,[
|
|
548
|
+
updated_at = excluded.updated_at`,[ct("idx","root:indexes/root.md"),"root","root",i.uri,"root",JSON.stringify({artifact_key:i.key,content_hash:i.hash??null,provenance:lt(i)}),r,r]);if(s){let d=ct("wiki","wiki/README.md");e.run(`INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
|
|
547
549
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
548
550
|
ON CONFLICT(path) DO UPDATE SET
|
|
549
551
|
title = excluded.title,
|
|
@@ -551,7 +553,7 @@ Pages should be concise, cited, and organized for both humans and agents.
|
|
|
551
553
|
content_hash = excluded.content_hash,
|
|
552
554
|
status = excluded.status,
|
|
553
555
|
metadata_json = excluded.metadata_json,
|
|
554
|
-
updated_at = excluded.updated_at`,[c,"wiki/README.md","Wiki",s.uri,s.hash??null,"active",JSON.stringify({artifact_key:s.key,provenance:rt(s)}),n,n]),ps(e,c,"Wiki",s,fr(),n)}}class mr{options;ensuredWorkspace;cachedConfig;constructor(e={}){this.options=e}get scope(){return this.options.scope??"global"}get workspace(){return this.ensuredWorkspace??st(this.options.scope,this.options.cwd)}ensureWorkspace(){if(!this.ensuredWorkspace)this.ensuredWorkspace=it(this.workspace.home);return this.ensuredWorkspace}jsonStorePath(){return this.ensureWorkspace().jsonStorePath}config(){if(!this.cachedConfig){let e=this.ensureWorkspace();this.cachedConfig=ot(e.configPath)}return this.cachedConfig}safetyPolicy(){return Ht(this.config(),this.ensureWorkspace())}artifactStore(){return gt(this.config(),this.ensureWorkspace())}storageContract(){return lr(this.config(),this.ensureWorkspace(),this.scope)}validateStorage(){return et(this.config(),this.ensureWorkspace())}paths(){let e=this.ensureWorkspace();return{ok:!0,scope:this.scope,home:e.home,config_path:e.configPath,json_store_path:e.jsonStorePath,knowledge_db_path:e.knowledgeDbPath,artifacts_dir:e.artifactsDir,indexes_dir:e.indexesDir,logs_dir:e.logsDir,runs_dir:e.runsDir,schemas_dir:e.schemasDir,wiki_dir:e.wikiDir,config:this.config(),message:e.home}}initDb(){return x(this.ensureWorkspace().knowledgeDbPath)}dbStats(){let e=this.ensureWorkspace();return x(e.knowledgeDbPath),ut(e.knowledgeDbPath)}async initWiki(){let e=this.ensureWorkspace();x(e.knowledgeDbPath);let t=await gr(this.artifactStore()),r=b(e.knowledgeDbPath);try{_r(r,t.artifacts),pr(r,t.artifacts)}finally{r.close()}return t}async ingestManifest(e){let t=this.ensureWorkspace();return Yt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}async ingestSource(e,t){let r=this.ensureWorkspace();return Zt({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t,config:this.config(),safetyPolicy:this.safetyPolicy()})}async resolveSource(e,t={}){let r=this.ensureWorkspace();return Oe({dbPath:r.knowledgeDbPath,sourceRef:e,purpose:t.purpose,limit:t.limit,safetyPolicy:this.safetyPolicy()})}async consumeOutbox(e){let t=this.ensureWorkspace();return Gt({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}reindexHealth(e={}){let t=this.ensureWorkspace();return rr({...e,dbPath:t.knowledgeDbPath,config:this.config()})}enqueueReindex(e={}){let t=this.ensureWorkspace();return Ze({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async refreshEmbeddings(e={}){let t=this.ensureWorkspace();return nr({...e,dbPath:t.knowledgeDbPath,config:this.config()})}providerStatus(e=process.env){return yt(this.config(),e)}modelRegistry(){return Xe(this.config())}embeddingStatus(){let e=this.ensureWorkspace();return Rt(e.knowledgeDbPath)}async indexEmbeddings(e={}){let t=this.ensureWorkspace();return me({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async semanticSearch(e){let t=this.ensureWorkspace();return he({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async search(e){let t=this.ensureWorkspace();return be({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async retrieveContext(e){let t=this.ensureWorkspace();return we({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async runPrompt(e){let t=this.ensureWorkspace();return Kt({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async webSearch(e){let t=this.ensureWorkspace();return ar({...e,dbPath:t.knowledgeDbPath,config:this.config(),safetyPolicy:this.safetyPolicy()})}}function hr(e={}){return new mr(e)}import{basename as hs}from"path";var ae={name:"@hasna/knowledge",version:"0.2.20",description:"Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",type:"module",bin:{knowledge:"bin/open-knowledge.js","open-knowledge":"bin/open-knowledge.js","open-knowledge-mcp":"bin/open-knowledge-mcp.js"},files:["bin","src","docs","LICENSE","README.md"],scripts:{test:"bun test","test:cli":"bun test tests/cli.test.ts",build:"bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",prepublishOnly:"bun run build",postinstall:"bun run build"},keywords:["knowledge","cli","agents","json","notes","local","store"],license:"Apache-2.0",publishConfig:{registry:"https://registry.npmjs.org",access:"public"},repository:{type:"git",url:"git+https://github.com/hasna/knowledge.git"},bugs:{url:"https://github.com/hasna/knowledge/issues"},author:"Hasna Inc. <hasna@example.com>",engines:{bun:">=1.0",node:">=18"},dependencies:{"@aws-sdk/client-s3":"^3.1063.0","@aws-sdk/credential-providers":"^3.1063.0","@ai-sdk/anthropic":"^3.0.81","@ai-sdk/deepseek":"^2.0.35","@ai-sdk/openai":"^3.0.68","@modelcontextprotocol/sdk":"^1.29.0",ai:"^6.0.197",zod:"^4.3.6"},devDependencies:{"@types/bun":"^1.3.14"}};var Er={debug:0,info:1,warn:2,error:3},Es=()=>{if(process.env.DEBUG)return"debug";if(process.env.LOG_LEVEL==="debug")return"debug";if(process.env.LOG_LEVEL==="warn")return"warn";if(process.env.LOG_LEVEL==="error")return"error";return"info"};function Y(e,t,r){if(Er[e]<Er[Es()])return;let n={debug:"[DEBUG]",info:"[INFO]",warn:"[WARN]",error:"[ERROR]"}[e],i=r?`${n} ${t} ${JSON.stringify(r)}`:`${n} ${t}`;if(e==="error")console.error(i);else console.error(i)}var yr=["add","list","get","delete","update","archive","restore","upsert","untag","export","prune","dedupe","stats","paths","storage","db","wiki","source","ingest","reindex","search","web","ask","build","embeddings","providers","safety","help"],kr={ls:"list",rm:"delete",edit:"update",unarchive:"restore",knowledge:"ask"};function ys(e){let t=[],r={};for(let n=0;n<e.length;n+=1){let i=e[n];if(!i.startsWith("-")){t.push(i);continue}switch(i){case"--json":r.json=!0;break;case"--yes":case"-y":r.yes=!0;break;case"--help":case"-h":r.help=!0;break;case"--version":case"-v":r.version=!0;break;case"--desc":r.desc=!0;break;case"--page":case"-p":r.page=Number(e[n+1]),n+=1;break;case"--limit":case"-l":r.limit=Number(e[n+1]),n+=1;break;case"--search":case"-s":r.search=e[n+1],n+=1;break;case"--sort":r.sort=e[n+1],n+=1;break;case"--id":r.id=e[n+1],n+=1;break;case"--store":r.store=e[n+1],n+=1;break;case"--title":r.title=e[n+1],n+=1;break;case"--content":r.content=e[n+1],n+=1;break;case"--url":r.url=e[n+1],n+=1;break;case"--tag":case"-t":r.tag=e[n+1],n+=1;break;case"--format":r.format=e[n+1],n+=1;break;case"--completions":r.completions=e[n+1],n+=1;break;case"--purpose":r.purpose=e[n+1],n+=1;break;case"--model":r.model=e[n+1],n+=1;break;case"--dimensions":r.dimensions=Number(e[n+1]),n+=1;break;case"--semantic":r.semantic=!0;break;case"--context":r.context=!0;break;case"--generate":r.generate=!0;break;case"--approve-write":r.approveWrite=!0;break;case"--provider":r.provider=e[n+1],n+=1;break;case"--domain":r.domain=[...r.domain??[],e[n+1]],n+=1;break;case"--file-results":r.fileResults=!0;break;case"--full":r.full=!0;break;case"--fake":r.fake=!0;break;case"--no-color":r.noColor=!0;break;case"--scope":r.scope=e[n+1],n+=1;break;case"--older-than":r.olderThan=Number(e[n+1]),n+=1;break;case"--empty":r.empty=!0;break;case"--archived":r.archived=!0;break;case"--include-archived":r.includeArchived=!0;break;default:throw Error(`Unknown flag: ${i}. Run 'open-knowledge --help' for valid options.`)}}return{positional:t,flags:r}}function ks(e){if(!e)return"";return kr[e]??e}function bs(e,t){let r=Array.from({length:e.length+1},()=>Array(t.length+1).fill(0));for(let n=0;n<=e.length;n+=1)r[n][0]=n;for(let n=0;n<=t.length;n+=1)r[0][n]=n;for(let n=1;n<=e.length;n+=1)for(let i=1;i<=t.length;i+=1){let s=e[n-1]===t[i-1]?0:1;r[n][i]=Math.min(r[n-1][i]+1,r[n][i-1]+1,r[n-1][i-1]+s)}return r[e.length][t.length]}function vs(e){if(!e)return"";let t=[...yr,...Object.keys(kr)],r="",n=Number.POSITIVE_INFINITY;for(let i of t){let s=bs(e,i);if(s<n)n=s,r=i}return n<=3?r:""}function ws(){return hs(process.argv[1]??"")==="knowledge"}function Ts(){console.log(`open-knowledge - local agent knowledge store
|
|
556
|
+
updated_at = excluded.updated_at`,[d,"wiki/README.md","Wiki",s.uri,s.hash??null,"active",JSON.stringify({artifact_key:s.key,provenance:lt(s)}),r,r]),Ws(e,d,"Wiki",s,Cn(),r)}}function Xs(e){if(!e)return;let t=e.trim().toLowerCase();if(t==="local"||t==="offline")return"local";if(t==="hosted"||t==="remote"||t==="knowledge.hasna.xyz")return"hosted";throw Error("Invalid setup mode. Use hosted or local.")}class Un{options;ensuredWorkspace;cachedConfig;constructor(e={}){this.options=e}get scope(){return this.options.scope??"global"}get workspace(){return this.ensuredWorkspace??pt(this.options.scope,this.options.cwd)}ensureWorkspace(){if(!this.ensuredWorkspace)this.ensuredWorkspace=gt(this.workspace.home);return this.ensuredWorkspace}jsonStorePath(){return this.ensureWorkspace().jsonStorePath}config(){if(!this.cachedConfig){let e=this.ensureWorkspace();this.cachedConfig=mt(e.configPath)}return this.cachedConfig}safetyPolicy(){return un(this.config(),this.ensureWorkspace())}artifactStore(){return St(this.config(),this.ensureWorkspace())}storageContract(){return In(this.config(),this.ensureWorkspace(),this.scope)}validateStorage(){return dt(this.config(),this.ensureWorkspace())}setup(e={}){let t=this.ensureWorkspace(),n=this.config(),r=Xs(e.mode)??n.mode,i=e.apiUrl?X(e.apiUrl):n.hosted?.api_url?X(n.hosted.api_url):null,s={...n,mode:r,hosted:{...n.hosted??{},...i?{api_url:i}:{}}};return ht(t.configPath,s),this.cachedConfig=s,{ok:!0,mode:r,api_url:s.hosted?.api_url??null,config_path:t.configPath,next:r==="hosted"?["open-knowledge auth login --api-key <key>","open-knowledge remote contracts --json"]:["open-knowledge search <query>","knowledge <prompt>"],message:`Set knowledge mode to ${r}`}}authStatus(e=process.env){return At(this.config(),e)}saveAuth(e,t=process.env){let n=e.apiUrl??this.config().hosted?.api_url;return Ot({api_key:e.apiKey,email:e.email,org_id:e.orgId,org_slug:e.orgSlug,user_id:e.userId,api_url:n},t)}clearAuth(e=process.env){return Nt(e)}remoteContract(){let e=this.storageContract();return wn({mode:this.config().mode,sourceSchemes:this.config().sources.allowed_schemes,storageType:e.artifact_store.type,artifactUriPrefix:e.artifact_store.uri_prefix})}remoteClient(e=process.env){return Ke.fromConfig(this.config(),e)}paths(){let e=this.ensureWorkspace();return{ok:!0,scope:this.scope,home:e.home,config_path:e.configPath,json_store_path:e.jsonStorePath,knowledge_db_path:e.knowledgeDbPath,artifacts_dir:e.artifactsDir,indexes_dir:e.indexesDir,logs_dir:e.logsDir,runs_dir:e.runsDir,schemas_dir:e.schemasDir,wiki_dir:e.wikiDir,config:this.config(),message:e.home}}initDb(){return S(this.ensureWorkspace().knowledgeDbPath)}dbStats(){let e=this.ensureWorkspace();return S(e.knowledgeDbPath),bt(e.knowledgeDbPath)}async initWiki(){let e=this.ensureWorkspace();S(e.knowledgeDbPath);let t=await Pn(this.artifactStore()),n=b(e.knowledgeDbPath);try{Ln(n,t.artifacts),Dn(n,t.artifacts)}finally{n.close()}return t}async ingestManifest(e){let t=this.ensureWorkspace();return fn({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}async ingestSource(e,t){let n=this.ensureWorkspace();return mn({dbPath:n.knowledgeDbPath,sourceRef:e,purpose:t,config:this.config(),safetyPolicy:this.safetyPolicy()})}async resolveSource(e,t={}){let n=this.ensureWorkspace();return Ce({dbPath:n.knowledgeDbPath,sourceRef:e,purpose:t.purpose,limit:t.limit,safetyPolicy:this.safetyPolicy()})}async consumeOutbox(e){let t=this.ensureWorkspace();return _n({dbPath:t.knowledgeDbPath,input:e,config:this.config(),safetyPolicy:this.safetyPolicy()})}reindexHealth(e={}){let t=this.ensureWorkspace();return yn({...e,dbPath:t.knowledgeDbPath,config:this.config()})}enqueueReindex(e={}){let t=this.ensureWorkspace();return ut({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async refreshEmbeddings(e={}){let t=this.ensureWorkspace();return kn({...e,dbPath:t.knowledgeDbPath,config:this.config()})}providerStatus(e=process.env){return Dt(this.config(),e)}modelRegistry(){return Ve(this.config())}embeddingStatus(){let e=this.ensureWorkspace();return $t(e.knowledgeDbPath)}async indexEmbeddings(e={}){let t=this.ensureWorkspace();return be({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async semanticSearch(e){let t=this.ensureWorkspace();return ve({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async search(e){let t=this.ensureWorkspace();return Re({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async retrieveContext(e){let t=this.ensureWorkspace();return Oe({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async runPrompt(e){let t=this.ensureWorkspace();return tn({...e,dbPath:t.knowledgeDbPath,config:this.config()})}async webSearch(e){let t=this.ensureWorkspace();return xn({...e,dbPath:t.knowledgeDbPath,config:this.config(),safetyPolicy:this.safetyPolicy()})}}function Kn(e={}){return new Un(e)}import{basename as qs}from"path";var le={name:"@hasna/knowledge",version:"0.2.21",description:"Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",type:"module",bin:{knowledge:"bin/open-knowledge.js","open-knowledge":"bin/open-knowledge.js","open-knowledge-mcp":"bin/open-knowledge-mcp.js"},files:["bin","src","docs","LICENSE","README.md"],scripts:{test:"bun test","test:cli":"bun test tests/cli.test.ts",build:"bun build --target=bun --outfile=bin/open-knowledge.js --minify --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/cli.ts && bun build --target=bun --outfile=bin/open-knowledge-mcp.js --external @modelcontextprotocol/sdk --external @aws-sdk/client-s3 --external @aws-sdk/credential-providers --external ai --external @ai-sdk/openai --external @ai-sdk/anthropic --external @ai-sdk/deepseek src/mcp.js",prepublishOnly:"bun run build",postinstall:"bun run build"},keywords:["knowledge","cli","agents","json","notes","local","store"],license:"Apache-2.0",publishConfig:{registry:"https://registry.npmjs.org",access:"public"},repository:{type:"git",url:"git+https://github.com/hasna/knowledge.git"},bugs:{url:"https://github.com/hasna/knowledge/issues"},author:"Hasna Inc. <hasna@example.com>",engines:{bun:">=1.0",node:">=18"},dependencies:{"@aws-sdk/client-s3":"^3.1063.0","@aws-sdk/credential-providers":"^3.1063.0","@ai-sdk/anthropic":"^3.0.81","@ai-sdk/deepseek":"^2.0.35","@ai-sdk/openai":"^3.0.68","@modelcontextprotocol/sdk":"^1.29.0",ai:"^6.0.197",zod:"^4.3.6"},devDependencies:{"@types/bun":"^1.3.14"}};var Mn={debug:0,info:1,warn:2,error:3},Hs=()=>{if(process.env.DEBUG)return"debug";if(process.env.LOG_LEVEL==="debug")return"debug";if(process.env.LOG_LEVEL==="warn")return"warn";if(process.env.LOG_LEVEL==="error")return"error";return"info"};function Q(e,t,n){if(Mn[e]<Mn[Hs()])return;let r={debug:"[DEBUG]",info:"[INFO]",warn:"[WARN]",error:"[ERROR]"}[e],i=n?`${r} ${t} ${JSON.stringify(n)}`:`${r} ${t}`;if(e==="error")console.error(i);else console.error(i)}var Fn=["add","list","get","delete","update","archive","restore","upsert","untag","export","prune","dedupe","stats","paths","setup","auth","remote","storage","db","wiki","source","ingest","reindex","search","web","ask","build","embeddings","providers","safety","help"],jn={ls:"list",rm:"delete",edit:"update",unarchive:"restore",knowledge:"ask"};function Bs(e){let t=[],n={};for(let r=0;r<e.length;r+=1){let i=e[r];if(!i.startsWith("-")){t.push(i);continue}switch(i){case"--json":n.json=!0;break;case"--yes":case"-y":n.yes=!0;break;case"--help":case"-h":n.help=!0;break;case"--version":case"-v":n.version=!0;break;case"--desc":n.desc=!0;break;case"--page":case"-p":n.page=Number(e[r+1]),r+=1;break;case"--limit":case"-l":n.limit=Number(e[r+1]),r+=1;break;case"--search":case"-s":n.search=e[r+1],r+=1;break;case"--sort":n.sort=e[r+1],r+=1;break;case"--id":n.id=e[r+1],r+=1;break;case"--store":n.store=e[r+1],r+=1;break;case"--title":n.title=e[r+1],r+=1;break;case"--content":n.content=e[r+1],r+=1;break;case"--url":n.url=e[r+1],r+=1;break;case"--tag":case"-t":n.tag=e[r+1],r+=1;break;case"--format":n.format=e[r+1],r+=1;break;case"--completions":n.completions=e[r+1],r+=1;break;case"--purpose":n.purpose=e[r+1],r+=1;break;case"--model":n.model=e[r+1],r+=1;break;case"--dimensions":n.dimensions=Number(e[r+1]),r+=1;break;case"--semantic":n.semantic=!0;break;case"--context":n.context=!0;break;case"--generate":n.generate=!0;break;case"--approve-write":n.approveWrite=!0;break;case"--provider":n.provider=e[r+1],r+=1;break;case"--mode":n.mode=e[r+1],r+=1;break;case"--api-url":n.apiUrl=e[r+1],r+=1;break;case"--api-key":n.apiKey=e[r+1],r+=1;break;case"--email":n.email=e[r+1],r+=1;break;case"--org":n.org=e[r+1],r+=1;break;case"--org-id":n.orgId=e[r+1],r+=1;break;case"--user-id":n.userId=e[r+1],r+=1;break;case"--domain":n.domain=[...n.domain??[],e[r+1]],r+=1;break;case"--file-results":n.fileResults=!0;break;case"--full":n.full=!0;break;case"--fake":n.fake=!0;break;case"--no-color":n.noColor=!0;break;case"--scope":n.scope=e[r+1],r+=1;break;case"--older-than":n.olderThan=Number(e[r+1]),r+=1;break;case"--empty":n.empty=!0;break;case"--archived":n.archived=!0;break;case"--include-archived":n.includeArchived=!0;break;default:throw Error(`Unknown flag: ${i}. Run 'open-knowledge --help' for valid options.`)}}return{positional:t,flags:n}}function zs(e){if(!e)return"";return jn[e]??e}function Js(e,t){let n=Array.from({length:e.length+1},()=>Array(t.length+1).fill(0));for(let r=0;r<=e.length;r+=1)n[r][0]=r;for(let r=0;r<=t.length;r+=1)n[0][r]=r;for(let r=1;r<=e.length;r+=1)for(let i=1;i<=t.length;i+=1){let s=e[r-1]===t[i-1]?0:1;n[r][i]=Math.min(n[r-1][i]+1,n[r][i-1]+1,n[r-1][i-1]+s)}return n[e.length][t.length]}function Gs(e){if(!e)return"";let t=[...Fn,...Object.keys(jn)],n="",r=Number.POSITIVE_INFINITY;for(let i of t){let s=Js(e,i);if(s<r)r=s,n=i}return r<=3?n:""}function Ys(){return qs(process.argv[1]??"")==="knowledge"}function Vs(){console.log(`open-knowledge - local agent knowledge store
|
|
555
557
|
|
|
556
558
|
Usage:
|
|
557
559
|
open-knowledge <command> [options]
|
|
@@ -571,6 +573,9 @@ Commands:
|
|
|
571
573
|
dedupe Remove duplicate items by title+content (requires --yes)
|
|
572
574
|
stats Show knowledge base statistics
|
|
573
575
|
paths Show resolved workspace/store paths
|
|
576
|
+
setup Configure local or hosted mode
|
|
577
|
+
auth login|whoami|logout Manage hosted API credentials
|
|
578
|
+
remote contracts|status Inspect hosted client contracts/readiness
|
|
574
579
|
storage status|validate Inspect local/S3 artifact storage contract
|
|
575
580
|
db init|stats Initialize or inspect local knowledge.db
|
|
576
581
|
wiki init Initialize scalable wiki/schema/index/log artifacts
|
|
@@ -597,6 +602,13 @@ Global Options:
|
|
|
597
602
|
--generate Call AI SDK text generation for ask/build
|
|
598
603
|
--approve-write Record approval intent for future durable wiki writes
|
|
599
604
|
--provider <name> Provider override for web search
|
|
605
|
+
--mode local|hosted Configure OSS local or hosted-aware mode
|
|
606
|
+
--api-url <url> Hosted API origin (or KNOWLEDGE_API_URL)
|
|
607
|
+
--api-key <key> Hosted API key for auth login
|
|
608
|
+
--email <email> Hosted account email metadata
|
|
609
|
+
--org <slug> Hosted organization slug metadata
|
|
610
|
+
--org-id <id> Hosted organization id metadata
|
|
611
|
+
--user-id <id> Hosted user id metadata
|
|
600
612
|
--domain <domain> Restrict provider web search to a domain
|
|
601
613
|
--file-results File web snippets as web source refs
|
|
602
614
|
--full Force full embedding index rebuild
|
|
@@ -637,5 +649,5 @@ Export Options:
|
|
|
637
649
|
|
|
638
650
|
Prune Options:
|
|
639
651
|
--older-than <days> Remove items older than N days
|
|
640
|
-
--empty Remove items with empty content`)}function
|
|
641
|
-
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--file-results)--file-results" "(--full)--full" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--provider)--provider[provider]:" "(--domain)--domain[domain]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l provider; complete -c open-knowledge -l domain; complete -c open-knowledge -l file-results; complete -c open-knowledge -l full; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let n=ks(t[0]),i=1;if(ws()&&n&&!yr.includes(n))n="ask",i=0;if(!n||r.help||n==="help"){xs(t[1]);return}let s=hr({scope:r.scope}),c=r.store;if(!c)if(r.scope==="project"||r.scope==="local")c=s.jsonStorePath();else c=Ue();if(n==="paths"){y(s.paths(),r.json);return}if(n==="storage"){let a=t[1]??"status";if(a==="status"){let d=s.storageContract(),o=s.validateStorage();y({ok:o.ok,...d,validation:o,message:`${d.storage_type} artifact storage at ${d.artifact_store.uri_prefix}`},r.json);return}if(a==="validate"){let d=s.validateStorage();y({ok:d.ok,validation:d,message:d.ok?"Storage contract valid":`Storage contract invalid: ${d.errors.join("; ")}`},r.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(n==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let o=s.initDb();y({ok:!0,...o,message:`Initialized ${o.path}`},r.json);return}let d=s.dbStats();y({ok:!0,path:s.workspace.knowledgeDbPath,...d,message:`knowledge.db schema v${d.schema_version}`},r.json);return}if(n==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let d=await s.initWiki();y({ok:!0,...d,message:`Initialized wiki layout in ${s.workspace.home}`},r.json);return}if(n==="safety"){let a=t[1]??"status",d=s.ensureWorkspace(),o=s.safetyPolicy();s.initDb();let l=b(d.knowledgeDbPath);try{if(a==="status"){y({ok:!0,mode:o.mode,workspace:d.home,allow_write_roots:o.allowWriteRoots,read_only_source_access:o.readOnlySourceAccess,network:o.network,redaction:o.redaction,approvals:o.approvals,message:`Safety policy: ${o.mode}`},r.json);return}if(a==="check"){let f=t[2]??"generated_write",g=t[3]??null,E;try{if(f==="web_search")Q(o),E={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!g)throw Error("safety check s3_read requires an s3:// target.");q(g,o),E={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"}}else E=zt(l,o,f,g);A(l,{event_type:"safety_check",action:f,target_uri:g,decision:E.decision==="allow"?"allow":"requires_approval",metadata:E}),y({ok:!0,...E,message:`Safety check ${E.decision}`},r.json);return}catch(h){throw A(l,{event_type:"safety_check",action:f,target_uri:g,decision:"deny",metadata:{error:h instanceof Error?h.message:String(h)}}),h}}if(a==="approve"){let f=t[2]??"generated_write",g=t[3]??null,E=Bt(l,{action:f,target_uri:g,reason:"local-cli approval",metadata:{scope:r.scope??"global"}});A(l,{event_type:"approval",action:f,target_uri:g,decision:"allow",metadata:{approval_id:E.id}}),y({ok:!0,...E,action:f,target_uri:g,message:`Approved ${f}`},r.json);return}if(a==="audit"){let f=l.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((g)=>({id:g.id,event_type:g.event_type,action:g.action,target_uri:g.target_uri,decision:g.decision,metadata:JSON.parse(g.metadata_json),created_at:g.created_at}));y({ok:!0,events:f,message:`${f.length} audit event(s)`},r.json);return}if(a==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let g=Te(f,o);if(g.findings.length>0)xe(l,{source_uri:"safety://redact",findings:g.findings,metadata:{command:"safety redact"}});A(l,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:g.findings.length>0?"redacted":"allow",metadata:{findings:g.findings.length}}),y({ok:!0,text:g.text,findings:g.findings,message:`Redacted ${g.findings.length} finding(s)`},r.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{l.close()}}if(n==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let d=t[2];if(!d)throw Error("Usage: open-knowledge source resolve <source-ref>");let o=await s.resolveSource(d,{purpose:r.purpose,limit:r.limit});y({ok:!0,...o,message:o.resolved?`Resolved ${o.source_ref} (${o.content.chunks_returned}/${o.content.chunks_total} chunks)`:`Source not indexed: ${d}`},r.json);return}if(n==="ingest"){let a=t[1]??"";if(a==="manifest"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let o=await s.ingestManifest(d);y({ok:!0,...o,message:`Ingested ${o.items_seen} manifest item(s)`},r.json);return}if(a==="source"){let d=t[2];if(!d)throw Error("Usage: open-knowledge ingest source <source-ref>");let o=await s.ingestSource(d,r.purpose);y({ok:!0,...o,message:`Ingested source ${o.source_ref} (${o.chunks_inserted} chunks)`},r.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(n==="reindex"){let a=t[1]??"status";if(a==="status"){let d=s.reindexHealth({modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`${d.missing_embeddings} chunk(s) missing embeddings`},r.json);return}if(a==="enqueue"){let d=s.enqueueReindex({modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`Queued ${d.enqueued} embedding refresh item(s)`},r.json);return}if(a==="embeddings"){let d=await s.refreshEmbeddings({full:r.full,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`Embedded ${d.indexed.chunks_embedded} chunk(s)`},r.json);return}if(a==="outbox"){let d=t[2];if(!d)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let o=await s.consumeOutbox(d);y({ok:!0,...o,message:`Consumed ${o.events_seen} outbox event(s)`},r.json);return}throw Error("Invalid reindex action. Use 'status', 'enqueue', 'embeddings', or 'outbox'.")}if(n==="embeddings"){let a=t[1]??"status";if(a==="status"){let d=s.embeddingStatus();y({ok:!0,...d,message:`${d.total_vector_entries} vector index entries`},r.json);return}if(a==="index"){let d=await s.indexEmbeddings({limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`Embedded ${d.chunks_embedded} chunk(s)`},r.json);return}if(a==="search"){let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge embeddings search <query>");let o=await s.semanticSearch({query:d,limit:r.limit,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...o,message:`${o.results.length} semantic result(s)`},r.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(n==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");if(r.context){let o=await s.retrieveContext({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...o,message:`${o.excerpts.length} context excerpt(s)`},r.json);return}let d=await s.search({query:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake});y({ok:!0,...d,message:`${d.results.length} search result(s)`},r.json);return}if(n==="web"){if((t[1]??"search")!=="search")throw Error("Invalid web action. Use 'search'.");let d=t.slice(2).join(" ");if(!d)throw Error("Usage: open-knowledge web search <query>");let o=await s.webSearch({query:d,limit:r.limit,modelRef:r.model,provider:r.provider,domains:r.domain,fake:r.fake,fileResults:r.fileResults});y({ok:!0,...o,message:`${o.sources.length} web source(s)`},r.json);return}if(n==="ask"||n==="build"){let a=t.slice(i).join(" ");if(!a)throw Error("Usage: open-knowledge ask <prompt>");let d=await s.runPrompt({prompt:a,limit:r.limit,semantic:r.semantic,modelRef:r.model,dimensions:r.dimensions,fake:r.fake,generate:r.generate,approveWrite:r.approveWrite});y({ok:!0,...d,message:d.generated?"Generated answer with citations":"Prepared citation context draft"},r.json);return}if(n==="providers"){let a=t[1]??"status";if(a==="status"){let d=s.providerStatus(),o=d.providers.filter((l)=>l.configured).length;y({ok:!0,...d,message:`${o}/${d.providers.length} provider credential(s) configured`},r.json);return}if(a==="models"){let d=s.modelRegistry();y({ok:!0,models:d,message:`${d.length} model alias(es)`},r.json);return}if(a==="check"){let d=t[2]??"default",o=X(d,s.config()),l=L(o),f=B(l.provider,s.config());y({ok:!0,target:d,model_ref:o,provider:l.provider,model:l.model,credential:f,message:`${l.provider} credentials configured`},r.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(Me(c),n==="add"){let a=t[1],d=t[2];if(!a||!d)throw Error("Usage: open-knowledge add <title> <content>");D(c,()=>{let o=P(c),l={id:je(),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};o.items.push(l),K(c,o),Y("info","Item added",{id:l.id,title:l.title}),y({ok:!0,item:l,message:`Added ${l.id}`},r.json)});return}if(n==="list"){if(r.format!==void 0&&r.format!=="table"&&r.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");D(c,()=>{let a=P(c),d=Number.isFinite(r.page)&&r.page>0?r.page:1,o=Number.isFinite(r.limit)&&r.limit>0?r.limit:20,l=r.search?String(r.search).toLowerCase():"",f=r.tag?String(r.tag).toLowerCase():"",g=r.format==="table"||!r.json&&!r.format&&Ss(r),E=r.json||r.format==="json",h=a.items;if(r.archived)h=h.filter((v)=>v.archived===!0);else if(!r.includeArchived)h=h.filter((v)=>!v.archived);if(l)h=h.filter((v)=>v.title.toLowerCase().includes(l)||v.content.toLowerCase().includes(l));if(f)h=h.filter((v)=>v.tags&&v.tags.map((Le)=>Le.toLowerCase()).includes(f));let{sorted:m,sort:T,direction:k}=Rs(h,r),p=(d-1)*o,R=m.slice(p,p+o),N=Math.max(1,Math.ceil(m.length/o));if(E){y({ok:!0,page:d,limit:o,total:m.length,total_pages:N,sort:T,direction:k,items:R},!0);return}if(R.length===0){y(`No items found (search=${l||"none"}, tag=${f||"none"})`,!1);return}if(g){let v=(F)=>F,Le=`${v("ID")} ${v("TITLE")} ${v("CREATED")} ${v("URL")} ${v("TAGS")}`;console.log(Le);for(let F of R)console.log(`${F.id} ${v(F.title)} ${F.created_at} ${F.url?v(F.url):""} ${F.tags?.length?v(`[${F.tags.join(", ")}]`):""}`);console.log(`Page ${d}/${N} | showing ${R.length} of ${m.length} | sort=${T} ${k} | search=${l||"none"} | tag=${f||"none"}`)}else{for(let v of R)console.log(`${v.id} ${v.title} ${v.created_at}${v.url?` ${v.url}`:""}${v.tags?.length?` [${v.tags.join(", ")}]`:""}`);console.log(`Page ${d}/${N} | showing ${R.length} of ${m.length} | sort=${T} ${k} | search=${l||"none"} | tag=${f||"none"}`)}});return}if(n==="get"){ce(r),D(c,()=>{let d=P(c).items.find((o)=>o.id===r.id||o.short_id===r.id);if(!d)throw Error(`Item not found: ${r.id}`);y({ok:!0,item:d,message:`${d.id}: ${d.title}`},r.json)});return}if(n==="update"){ce(r),D(c,()=>{let a=P(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];if(r.title!==void 0)o.title=r.title;if(r.content!==void 0)o.content=r.content;if(r.url!==void 0)o.url=r.url;if(r.tag!==void 0){if(o.tags=o.tags||[],!o.tags.map((l)=>l.toLowerCase()).includes(r.tag.toLowerCase()))o.tags.push(r.tag)}o.updated_at=new Date().toISOString(),a.items[d]=o,K(c,a),y({ok:!0,item:o,message:`Updated ${o.id}`},r.json)});return}if(n==="archive"||n==="restore"){ce(r),D(c,()=>{let a=P(c),d=a.items.findIndex((l)=>l.id===r.id||l.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d];o.archived=n==="archive",o.updated_at=new Date().toISOString(),a.items[d]=o,K(c,a),y({ok:!0,item:o,message:`${n==="archive"?"Archived":"Restored"} ${o.id}`},r.json)});return}if(n==="untag"){if(ce(r),!r.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");D(c,()=>{let a=P(c),d=a.items.findIndex((f)=>f.id===r.id||f.short_id===r.id);if(d===-1)throw Error(`Item not found: ${r.id}`);let o=a.items[d],l=o.tags?.length??0;o.tags=(o.tags??[]).filter((f)=>f.toLowerCase()!==r.tag.toLowerCase()),o.updated_at=new Date().toISOString(),a.items[d]=o,K(c,a),y({ok:!0,item:o,removed:l-o.tags.length,message:`Removed tag from ${o.id}`},r.json)});return}if(n==="upsert"){let a=r.title??t[1],d=r.content??t[2];D(c,()=>{let o=P(c),l=r.id?o.items.findIndex((E)=>E.id===r.id||E.short_id===r.id):-1,f=new Date().toISOString();if(l===-1){if(!a||!d)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let E=r.id??je(),h={id:E,short_id:dt(E),title:a,content:d,url:r.url??null,tags:r.tag?[r.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};o.items.push(h),K(c,o),y({ok:!0,created:!0,item:h,message:`Upserted ${h.id}`},r.json);return}let g=o.items[l];if(a!==void 0)g.title=a;if(d!==void 0)g.content=d;if(r.url!==void 0)g.url=r.url;if(r.tag!==void 0){if(g.tags=g.tags||[],!g.tags.map((E)=>E.toLowerCase()).includes(r.tag.toLowerCase()))g.tags.push(r.tag)}g.updated_at=f,o.items[l]=g,K(c,o),y({ok:!0,created:!1,item:g,message:`Upserted ${g.id}`},r.json)});return}if(n==="delete"){if(ce(r),!r.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");D(c,()=>{let a=P(c),d=a.items.length;a.items=a.items.filter((l)=>l.id!==r.id&&l.short_id!==r.id);let o=d!==a.items.length;if(K(c,a),!o)throw Error(`Item not found: ${r.id}`);Y("info","Item deleted",{id:r.id}),y({ok:!0,deleted_id:r.id,message:`Deleted ${r.id}`},r.json)});return}if(n==="export"){let a=r.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");D(c,()=>{let d=P(c);if(a==="jsonl")for(let o of d.items)console.log(JSON.stringify(o));else y({ok:!0,items:d.items},r.json)});return}if(n==="prune"){if(!r.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");D(c,()=>{let a=P(c),d=a.items.length;if(r.olderThan!==void 0){let l=new Date;l.setDate(l.getDate()-r.olderThan),a.items=a.items.filter((f)=>new Date(f.created_at)>=l)}if(r.empty)a.items=a.items.filter((l)=>l.content.trim().length>0);let o=d-a.items.length;K(c,a),Y("info","Prune completed",{pruned:o,remaining:a.items.length}),y({ok:!0,pruned:o,remaining:a.items.length,message:`Pruned ${o} item(s)`},r.json)});return}if(n==="dedupe"){if(!r.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");D(c,()=>{let a=P(c),d=new Set,o=a.items.length;a.items=a.items.filter((f)=>{let g=`${f.title}\x00${f.content}`;if(d.has(g))return!1;return d.add(g),!0});let l=o-a.items.length;K(c,a),Y("info","Dedupe completed",{removed:l,remaining:a.items.length}),y({ok:!0,removed:l,remaining:a.items.length,message:`Dedupe removed ${l} duplicate(s)`},r.json)});return}if(n==="stats"){D(c,()=>{let a=P(c),d=a.items.filter((k)=>!k.archived),o=d.length,l=a.items.length-o,f=d.filter((k)=>k.url).length,g=d.filter((k)=>k.tags&&k.tags.length>0).length,E=o>0?d.map((k)=>k.created_at).sort()[0]:null,h=o>0?d.map((k)=>k.created_at).sort()[o-1]:null,m={};for(let k of d)for(let p of k.tags||[])m[p]=(m[p]||0)+1;let T=Object.entries(m).sort((k,p)=>p[1]-k[1]).slice(0,5).map(([k,p])=>({tag:k,count:p}));y({ok:!0,total:o,archived:l,with_url:f,with_tags:g,oldest:E,newest:h,top_tags:T,message:`${o} items | ${f} with URL | ${g} with tags`},r.json)});return}let u=vs(t[0]),_=u?` Did you mean '${u}'?`:"";throw Y("warn","Unknown command",{input:t[0],suggestion:u}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)Os(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);Y("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{vs as suggestCommand,Rs as sortItems,Os as run,ys as parseArgs};
|
|
652
|
+
--empty Remove items with empty content`)}function Qs(e){if(e==="add"){console.log("Usage: open-knowledge add <title> <content> [--url <url>] [-t <tag>] [--json]");return}if(e==="list"||e==="ls"){console.log("Usage: open-knowledge list|ls [--format table|json] [-p <page>] [-l <limit>] [-s <search>] [-t <tag>] [--sort created|title] [--desc] [--json]");return}if(e==="get"){console.log("Usage: open-knowledge get --id <id> [--json]");return}if(e==="update"||e==="edit"){console.log("Usage: open-knowledge update|edit --id <id> [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="archive"){console.log("Usage: open-knowledge archive --id <id> [--json]");return}if(e==="restore"||e==="unarchive"){console.log("Usage: open-knowledge restore|unarchive --id <id> [--json]");return}if(e==="upsert"){console.log("Usage: open-knowledge upsert [title] [content] [--id <id>] [--title <title>] [--content <content>] [--url <url>] [-t <tag>] [--json]");return}if(e==="untag"){console.log("Usage: open-knowledge untag --id <id> -t <tag> [--json]");return}if(e==="delete"||e==="rm"){console.log("Usage: open-knowledge delete|rm --id <id> -y [--json]");return}if(e==="export"){console.log("Usage: open-knowledge export [--format jsonl] [--json]");return}if(e==="prune"){console.log("Usage: open-knowledge prune --yes [--older-than <days>] [--empty] [--json]");return}if(e==="dedupe"){console.log("Usage: open-knowledge dedupe --yes [--json]");return}if(e==="stats"){console.log("Usage: open-knowledge stats [--json]");return}if(e==="paths"){console.log("Usage: open-knowledge paths [--scope local|global|project] [--json]");return}if(e==="setup"){console.log("Usage: open-knowledge setup --mode local|hosted [--api-url https://...] [--scope local|global|project] [--json]");return}if(e==="auth"){console.log("Usage: open-knowledge auth login|whoami|logout [--api-key <key>] [--email <email>] [--org <slug>] [--api-url https://...] [--scope local|global|project] [--json]");return}if(e==="remote"){console.log("Usage: open-knowledge remote contracts|status [--scope local|global|project] [--json]");return}if(e==="storage"){console.log("Usage: open-knowledge storage status|validate [--scope local|global|project] [--json]");return}if(e==="db"){console.log("Usage: open-knowledge db init|stats [--scope local|global|project] [--json]");return}if(e==="wiki"){console.log("Usage: open-knowledge wiki init [--scope local|global|project] [--json]");return}if(e==="source"){console.log("Usage: open-knowledge source resolve <source-ref> [--purpose knowledge_answer|knowledge_index] [--limit <n>] [--scope local|global|project] [--json]");return}if(e==="ingest"){console.log("Usage: open-knowledge ingest manifest <file|s3://bucket/key> | source <source-ref> [--purpose knowledge_index] [--scope local|global|project] [--json]");return}if(e==="reindex"){console.log("Usage: open-knowledge reindex status|enqueue|embeddings|outbox [file|s3://bucket/key] [--full] [--fake] [--scope local|global|project] [--json]");return}if(e==="search"){console.log("Usage: open-knowledge search <query> [--context] [--semantic] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="web"){console.log("Usage: open-knowledge web search <query> [--provider openai|anthropic] [--model provider:model] [--domain <domain>] [--file-results] [--fake] [--scope local|global|project] [--json]");return}if(e==="ask"||e==="build"||e==="knowledge"){console.log("Usage: open-knowledge ask|build <prompt> [--generate] [--semantic] [--model default|provider:model] [--approve-write] [--scope local|global|project] [--json]");return}if(e==="embeddings"){console.log("Usage: open-knowledge embeddings status|index|search [query] [--model openai:text-embedding-3-small] [--limit <n>] [--dimensions <n>] [--fake] [--scope local|global|project] [--json]");return}if(e==="providers"){console.log("Usage: open-knowledge providers status|models|check [provider|model-alias] [--scope local|global|project] [--json]");return}if(e==="safety"){console.log("Usage: open-knowledge safety status|check|approve|audit|redact [args] [--scope local|global|project] [--json]");return}Vs()}function Zs(e){if(e.noColor||process.env.NO_COLOR)return!1;if(process.env.FORCE_COLOR)return!0;return process.stdout.isTTY===!0}function y(e,t,n){if(t){console.log(JSON.stringify(e,null,2));return}if(typeof e==="string"){console.log(e);return}console.log(e.message??JSON.stringify(e,null,2))}function _e(e){if(!e.id)throw Error("Missing required --id. Example: open-knowledge get --id <id>")}function eo(e,t){let n=t.sort??"created";if(n!=="created"&&n!=="title")throw Error("Invalid --sort value. Use 'created' or 'title'.");let r=[...e].sort((i,s)=>{if(n==="title")return i.title.localeCompare(s.title);return i.created_at.localeCompare(s.created_at)});if(t.desc)r.reverse();return{sorted:r,sort:n,direction:t.desc?"desc":"asc"}}async function to(e){let{positional:t,flags:n}=Bs(e);if(Q("debug","CLI invoked",{command:t[0],flags:{json:n.json,store:n.store}}),n.version){console.log(n.json?JSON.stringify({name:le.name,version:le.version},null,2):`${le.name} ${le.version}`);return}if(n.completions){let a=n.completions;if(a==="bash")console.log('_open_knowledge() { local cur; cur="${COMP_WORDS[COMP_CWORD]}"; COMPREPLY=($(compgen -W "add list get update archive restore upsert untag delete export prune dedupe stats paths setup auth remote storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge --json --yes --help --version --desc --page --limit --search --sort --id --store --title --content --url --tag --format --completions --purpose --model --dimensions --semantic --context --generate --approve-write --provider --mode --api-url --api-key --email --org --org-id --user-id --domain --file-results --full --fake --no-color --scope --archived --include-archived" -- "$cur")); }; complete -F _open_knowledge open-knowledge');else if(a==="zsh")console.log(`#compdef open-knowledge
|
|
653
|
+
_open_knowledge() { _arguments -C "1: :(add list get update archive restore upsert untag delete export prune dedupe stats paths setup auth remote storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge)" "(--json)--json" "(--yes)-y" "(--help)--help" "(--version)--version" "(--desc)--desc" "(--archived)--archived" "(--include-archived)--include-archived" "(--semantic)--semantic" "(--context)--context" "(--generate)--generate" "(--approve-write)--approve-write" "(--file-results)--file-results" "(--full)--full" "(--fake)--fake" "(-p --page)"{-p,--page}"[page number]:number:" "(-l --limit)"{-l,--limit}"[items per page]:number:" "(-s --search)"{-s,--search}"[search text]:text:" "(--sort)--sort"{created,title}:" "(--id)--id[item id]:id:" "(--store)--store[store path]:path:" "(--title)--title[new title]:" "(--content)--content[new content]:" "(--url)--url[source url]:" "(-t --tag)"{-t,--tag}"[tag]:tag:" "(--format)--format[json|jsonl]:" "(--completions)--completions[output completions]:shell:(bash zsh fish):" "(--purpose)--purpose[purpose]:" "(--model)--model[model ref]:" "(--dimensions)--dimensions[embedding dimensions]:number:" "(--provider)--provider[provider]:" "(--mode)--mode"{local,hosted}:" "(--api-url)--api-url[hosted API URL]:" "(--api-key)--api-key[hosted API key]:" "(--email)--email[email]:" "(--org)--org[org slug]:" "(--org-id)--org-id[org id]:" "(--user-id)--user-id[user id]:" "(--domain)--domain[domain]:" "(--no-color)--no-color[disable color]" "(--scope)--scope"{local,global,project}:" }; _open_knowledge`);else if(a==="fish")console.log('complete -c open-knowledge -f; complete -c open-knowledge -a "add list get update archive restore upsert untag delete export prune dedupe stats paths setup auth remote storage db wiki source ingest reindex search web ask build embeddings providers safety help ls rm edit unarchive knowledge"; complete -c open-knowledge -l json; complete -c open-knowledge -l yes -s y; complete -c open-knowledge -l help -s h; complete -c open-knowledge -l version -s v; complete -c open-knowledge -l desc; complete -c open-knowledge -l archived; complete -c open-knowledge -l include-archived; complete -c open-knowledge -l semantic; complete -c open-knowledge -l context; complete -c open-knowledge -l generate; complete -c open-knowledge -l approve-write; complete -c open-knowledge -l provider; complete -c open-knowledge -l mode; complete -c open-knowledge -l api-url; complete -c open-knowledge -l api-key; complete -c open-knowledge -l email; complete -c open-knowledge -l org; complete -c open-knowledge -l org-id; complete -c open-knowledge -l user-id; complete -c open-knowledge -l domain; complete -c open-knowledge -l file-results; complete -c open-knowledge -l full; complete -c open-knowledge -l fake; complete -c open-knowledge -s p -l page; complete -c open-knowledge -s l -l limit; complete -c open-knowledge -s s -l search; complete -c open-knowledge -l sort; complete -c open-knowledge -l id; complete -c open-knowledge -l store; complete -c open-knowledge -l title; complete -c open-knowledge -l content; complete -c open-knowledge -l url; complete -c open-knowledge -s t -l tag; complete -c open-knowledge -l format; complete -c open-knowledge -l completions; complete -c open-knowledge -l purpose; complete -c open-knowledge -l model; complete -c open-knowledge -l dimensions; complete -c open-knowledge -l no-color; complete -c open-knowledge -l scope -a "local global project"');else throw Error("Invalid --completions value. Use 'bash', 'zsh', or 'fish'.");return}let r=zs(t[0]),i=1;if(Ys()&&r&&!Fn.includes(r))r="ask",i=0;if(!r||n.help||r==="help"){Qs(t[1]);return}let s=Kn({scope:n.scope}),d=n.store;if(!d)if(n.scope==="project"||n.scope==="local")d=s.jsonStorePath();else d=$e();if(r==="paths"){y(s.paths(),n.json);return}if(r==="setup"){let a=s.setup({mode:n.mode,apiUrl:n.apiUrl});y(a,n.json);return}if(r==="auth"){let a=t[1]??"whoami";if(a==="whoami"||a==="status"){let u=s.authStatus(process.env);y({ok:!0,...u,message:u.authenticated?`Authenticated via ${u.source}`:"Not authenticated"},n.json);return}if(a==="login"){let u=n.apiKey??process.env.KNOWLEDGE_API_KEY??process.env.HASNA_KNOWLEDGE_API_KEY;if(!u)throw Error("Usage: open-knowledge auth login --api-key <key> [--email <email>]");let o=s.saveAuth({apiKey:u,email:n.email,orgSlug:n.org,orgId:n.orgId,userId:n.userId,apiUrl:n.apiUrl},process.env);y({ok:!0,authenticated:!0,email:o.email??null,org_slug:o.org_slug??null,api_url:o.api_url??s.authStatus(process.env).api_url,auth_path:s.authStatus(process.env).auth_path,message:`Saved hosted credentials for ${o.email??"API key"}`},n.json);return}if(a==="logout"){let u=s.clearAuth(process.env);y({ok:!0,removed:u,message:u?"Removed hosted credentials":"No hosted credentials found"},n.json);return}throw Error("Invalid auth action. Use 'login', 'whoami', or 'logout'.")}if(r==="remote"){let a=t[1]??"status";if(a==="contracts"||a==="contract"){let u=s.authStatus(process.env);y({ok:!0,authenticated:u.authenticated,api_url:u.api_url,contract:s.remoteContract(),message:`Remote contract v${s.remoteContract().contract_version}`},n.json);return}if(a==="status"){let u=s.authStatus(process.env),o=s.remoteContract();y({ok:!0,mode:s.config().mode,authenticated:u.authenticated,auth_source:u.source,api_url:u.api_url,client_ready:Boolean(s.remoteClient(process.env)),contract_version:o.contract_version,capabilities:o.capabilities,message:u.authenticated?`Remote client ready for ${u.api_url}`:"Remote client not authenticated"},n.json);return}throw Error("Invalid remote action. Use 'contracts' or 'status'.")}if(r==="storage"){let a=t[1]??"status";if(a==="status"){let u=s.storageContract(),o=s.validateStorage();y({ok:o.ok,...u,validation:o,message:`${u.storage_type} artifact storage at ${u.artifact_store.uri_prefix}`},n.json);return}if(a==="validate"){let u=s.validateStorage();y({ok:u.ok,validation:u,message:u.ok?"Storage contract valid":`Storage contract invalid: ${u.errors.join("; ")}`},n.json);return}throw Error("Invalid storage action. Use 'status' or 'validate'.")}if(r==="db"){let a=t[1]??"init";if(a!=="init"&&a!=="stats")throw Error("Invalid db action. Use 'init' or 'stats'.");if(a==="init"){let o=s.initDb();y({ok:!0,...o,message:`Initialized ${o.path}`},n.json);return}let u=s.dbStats();y({ok:!0,path:s.workspace.knowledgeDbPath,...u,message:`knowledge.db schema v${u.schema_version}`},n.json);return}if(r==="wiki"){if((t[1]??"init")!=="init")throw Error("Invalid wiki action. Use 'init'.");let u=await s.initWiki();y({ok:!0,...u,message:`Initialized wiki layout in ${s.workspace.home}`},n.json);return}if(r==="safety"){let a=t[1]??"status",u=s.ensureWorkspace(),o=s.safetyPolicy();s.initDb();let l=b(u.knowledgeDbPath);try{if(a==="status"){y({ok:!0,mode:o.mode,workspace:u.home,allow_write_roots:o.allowWriteRoots,read_only_source_access:o.readOnlySourceAccess,network:o.network,redaction:o.redaction,approvals:o.approvals,message:`Safety policy: ${o.mode}`},n.json);return}if(a==="check"){let f=t[2]??"generated_write",g=t[3]??null,E;try{if(f==="web_search")ee(o),E={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"};else if(f==="s3_read"){if(!g)throw Error("safety check s3_read requires an s3:// target.");B(g,o),E={action:f,target_uri:g,approval_required:!1,approved:!0,decision:"allow"}}else E=cn(l,o,f,g);A(l,{event_type:"safety_check",action:f,target_uri:g,decision:E.decision==="allow"?"allow":"requires_approval",metadata:E}),y({ok:!0,...E,message:`Safety check ${E.decision}`},n.json);return}catch(h){throw A(l,{event_type:"safety_check",action:f,target_uri:g,decision:"deny",metadata:{error:h instanceof Error?h.message:String(h)}}),h}}if(a==="approve"){let f=t[2]??"generated_write",g=t[3]??null,E=dn(l,{action:f,target_uri:g,reason:"local-cli approval",metadata:{scope:n.scope??"global"}});A(l,{event_type:"approval",action:f,target_uri:g,decision:"allow",metadata:{approval_id:E.id}}),y({ok:!0,...E,action:f,target_uri:g,message:`Approved ${f}`},n.json);return}if(a==="audit"){let f=l.query("SELECT id, event_type, action, target_uri, decision, metadata_json, created_at FROM audit_events ORDER BY created_at DESC LIMIT 50").all().map((g)=>({id:g.id,event_type:g.event_type,action:g.action,target_uri:g.target_uri,decision:g.decision,metadata:JSON.parse(g.metadata_json),created_at:g.created_at}));y({ok:!0,events:f,message:`${f.length} audit event(s)`},n.json);return}if(a==="redact"){let f=t.slice(2).join(" ");if(!f)throw Error("Usage: open-knowledge safety redact <text>");let g=Ne(f,o);if(g.findings.length>0)Ae(l,{source_uri:"safety://redact",findings:g.findings,metadata:{command:"safety redact"}});A(l,{event_type:"redaction",action:"safety_redact",target_uri:"safety://redact",decision:g.findings.length>0?"redacted":"allow",metadata:{findings:g.findings.length}}),y({ok:!0,text:g.text,findings:g.findings,message:`Redacted ${g.findings.length} finding(s)`},n.json);return}throw Error("Invalid safety action. Use 'status', 'check', 'approve', 'audit', or 'redact'.")}finally{l.close()}}if(r==="source"){if((t[1]??"")!=="resolve")throw Error("Invalid source action. Use 'resolve'.");let u=t[2];if(!u)throw Error("Usage: open-knowledge source resolve <source-ref>");let o=await s.resolveSource(u,{purpose:n.purpose,limit:n.limit});y({ok:!0,...o,message:o.resolved?`Resolved ${o.source_ref} (${o.content.chunks_returned}/${o.content.chunks_total} chunks)`:`Source not indexed: ${u}`},n.json);return}if(r==="ingest"){let a=t[1]??"";if(a==="manifest"){let u=t[2];if(!u)throw Error("Usage: open-knowledge ingest manifest <file|s3://bucket/key>");let o=await s.ingestManifest(u);y({ok:!0,...o,message:`Ingested ${o.items_seen} manifest item(s)`},n.json);return}if(a==="source"){let u=t[2];if(!u)throw Error("Usage: open-knowledge ingest source <source-ref>");let o=await s.ingestSource(u,n.purpose);y({ok:!0,...o,message:`Ingested source ${o.source_ref} (${o.chunks_inserted} chunks)`},n.json);return}throw Error("Invalid ingest action. Use 'manifest' or 'source'.")}if(r==="reindex"){let a=t[1]??"status";if(a==="status"){let u=s.reindexHealth({modelRef:n.model,dimensions:n.dimensions,fake:n.fake});y({ok:!0,...u,message:`${u.missing_embeddings} chunk(s) missing embeddings`},n.json);return}if(a==="enqueue"){let u=s.enqueueReindex({modelRef:n.model,dimensions:n.dimensions,fake:n.fake});y({ok:!0,...u,message:`Queued ${u.enqueued} embedding refresh item(s)`},n.json);return}if(a==="embeddings"){let u=await s.refreshEmbeddings({full:n.full,limit:n.limit,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});y({ok:!0,...u,message:`Embedded ${u.indexed.chunks_embedded} chunk(s)`},n.json);return}if(a==="outbox"){let u=t[2];if(!u)throw Error("Usage: open-knowledge reindex outbox <file|s3://bucket/key>");let o=await s.consumeOutbox(u);y({ok:!0,...o,message:`Consumed ${o.events_seen} outbox event(s)`},n.json);return}throw Error("Invalid reindex action. Use 'status', 'enqueue', 'embeddings', or 'outbox'.")}if(r==="embeddings"){let a=t[1]??"status";if(a==="status"){let u=s.embeddingStatus();y({ok:!0,...u,message:`${u.total_vector_entries} vector index entries`},n.json);return}if(a==="index"){let u=await s.indexEmbeddings({limit:n.limit,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});y({ok:!0,...u,message:`Embedded ${u.chunks_embedded} chunk(s)`},n.json);return}if(a==="search"){let u=t.slice(2).join(" ");if(!u)throw Error("Usage: open-knowledge embeddings search <query>");let o=await s.semanticSearch({query:u,limit:n.limit,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});y({ok:!0,...o,message:`${o.results.length} semantic result(s)`},n.json);return}throw Error("Invalid embeddings action. Use 'status', 'index', or 'search'.")}if(r==="search"){let a=t.slice(1).join(" ");if(!a)throw Error("Usage: open-knowledge search <query>");if(n.context){let o=await s.retrieveContext({query:a,limit:n.limit,semantic:n.semantic,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});y({ok:!0,...o,message:`${o.excerpts.length} context excerpt(s)`},n.json);return}let u=await s.search({query:a,limit:n.limit,semantic:n.semantic,modelRef:n.model,dimensions:n.dimensions,fake:n.fake});y({ok:!0,...u,message:`${u.results.length} search result(s)`},n.json);return}if(r==="web"){if((t[1]??"search")!=="search")throw Error("Invalid web action. Use 'search'.");let u=t.slice(2).join(" ");if(!u)throw Error("Usage: open-knowledge web search <query>");let o=await s.webSearch({query:u,limit:n.limit,modelRef:n.model,provider:n.provider,domains:n.domain,fake:n.fake,fileResults:n.fileResults});y({ok:!0,...o,message:`${o.sources.length} web source(s)`},n.json);return}if(r==="ask"||r==="build"){let a=t.slice(i).join(" ");if(!a)throw Error("Usage: open-knowledge ask <prompt>");let u=await s.runPrompt({prompt:a,limit:n.limit,semantic:n.semantic,modelRef:n.model,dimensions:n.dimensions,fake:n.fake,generate:n.generate,approveWrite:n.approveWrite});y({ok:!0,...u,message:u.generated?"Generated answer with citations":"Prepared citation context draft"},n.json);return}if(r==="providers"){let a=t[1]??"status";if(a==="status"){let u=s.providerStatus(),o=u.providers.filter((l)=>l.configured).length;y({ok:!0,...u,message:`${o}/${u.providers.length} provider credential(s) configured`},n.json);return}if(a==="models"){let u=s.modelRegistry();y({ok:!0,models:u,message:`${u.length} model alias(es)`},n.json);return}if(a==="check"){let u=t[2]??"default",o=q(u,s.config()),l=L(o),f=J(l.provider,s.config());y({ok:!0,target:u,model_ref:o,provider:l.provider,model:l.model,credential:f,message:`${l.provider} credentials configured`},n.json);return}throw Error("Invalid providers action. Use 'status', 'models', or 'check'.")}if(qe(d),r==="add"){let a=t[1],u=t[2];if(!a||!u)throw Error("Usage: open-knowledge add <title> <content>");D(d,()=>{let o=P(d),l={id:He(),title:a,content:u,url:n.url??null,tags:n.tag?[n.tag]:[],created_at:new Date().toISOString(),updated_at:new Date().toISOString()};o.items.push(l),j(d,o),Q("info","Item added",{id:l.id,title:l.title}),y({ok:!0,item:l,message:`Added ${l.id}`},n.json)});return}if(r==="list"){if(n.format!==void 0&&n.format!=="table"&&n.format!=="json")throw Error("Invalid --format value for list. Use 'table' or 'json'.");D(d,()=>{let a=P(d),u=Number.isFinite(n.page)&&n.page>0?n.page:1,o=Number.isFinite(n.limit)&&n.limit>0?n.limit:20,l=n.search?String(n.search).toLowerCase():"",f=n.tag?String(n.tag).toLowerCase():"",g=n.format==="table"||!n.json&&!n.format&&Zs(n),E=n.json||n.format==="json",h=a.items;if(n.archived)h=h.filter((v)=>v.archived===!0);else if(!n.includeArchived)h=h.filter((v)=>!v.archived);if(l)h=h.filter((v)=>v.title.toLowerCase().includes(l)||v.content.toLowerCase().includes(l));if(f)h=h.filter((v)=>v.tags&&v.tags.map((Fe)=>Fe.toLowerCase()).includes(f));let{sorted:m,sort:T,direction:k}=eo(h,n),p=(u-1)*o,x=m.slice(p,p+o),N=Math.max(1,Math.ceil(m.length/o));if(E){y({ok:!0,page:u,limit:o,total:m.length,total_pages:N,sort:T,direction:k,items:x},!0);return}if(x.length===0){y(`No items found (search=${l||"none"}, tag=${f||"none"})`,!1);return}if(g){let v=(W)=>W,Fe=`${v("ID")} ${v("TITLE")} ${v("CREATED")} ${v("URL")} ${v("TAGS")}`;console.log(Fe);for(let W of x)console.log(`${W.id} ${v(W.title)} ${W.created_at} ${W.url?v(W.url):""} ${W.tags?.length?v(`[${W.tags.join(", ")}]`):""}`);console.log(`Page ${u}/${N} | showing ${x.length} of ${m.length} | sort=${T} ${k} | search=${l||"none"} | tag=${f||"none"}`)}else{for(let v of x)console.log(`${v.id} ${v.title} ${v.created_at}${v.url?` ${v.url}`:""}${v.tags?.length?` [${v.tags.join(", ")}]`:""}`);console.log(`Page ${u}/${N} | showing ${x.length} of ${m.length} | sort=${T} ${k} | search=${l||"none"} | tag=${f||"none"}`)}});return}if(r==="get"){_e(n),D(d,()=>{let u=P(d).items.find((o)=>o.id===n.id||o.short_id===n.id);if(!u)throw Error(`Item not found: ${n.id}`);y({ok:!0,item:u,message:`${u.id}: ${u.title}`},n.json)});return}if(r==="update"){_e(n),D(d,()=>{let a=P(d),u=a.items.findIndex((l)=>l.id===n.id||l.short_id===n.id);if(u===-1)throw Error(`Item not found: ${n.id}`);let o=a.items[u];if(n.title!==void 0)o.title=n.title;if(n.content!==void 0)o.content=n.content;if(n.url!==void 0)o.url=n.url;if(n.tag!==void 0){if(o.tags=o.tags||[],!o.tags.map((l)=>l.toLowerCase()).includes(n.tag.toLowerCase()))o.tags.push(n.tag)}o.updated_at=new Date().toISOString(),a.items[u]=o,j(d,a),y({ok:!0,item:o,message:`Updated ${o.id}`},n.json)});return}if(r==="archive"||r==="restore"){_e(n),D(d,()=>{let a=P(d),u=a.items.findIndex((l)=>l.id===n.id||l.short_id===n.id);if(u===-1)throw Error(`Item not found: ${n.id}`);let o=a.items[u];o.archived=r==="archive",o.updated_at=new Date().toISOString(),a.items[u]=o,j(d,a),y({ok:!0,item:o,message:`${r==="archive"?"Archived":"Restored"} ${o.id}`},n.json)});return}if(r==="untag"){if(_e(n),!n.tag)throw Error("Missing required --tag. Example: open-knowledge untag --id <id> -t <tag>");D(d,()=>{let a=P(d),u=a.items.findIndex((f)=>f.id===n.id||f.short_id===n.id);if(u===-1)throw Error(`Item not found: ${n.id}`);let o=a.items[u],l=o.tags?.length??0;o.tags=(o.tags??[]).filter((f)=>f.toLowerCase()!==n.tag.toLowerCase()),o.updated_at=new Date().toISOString(),a.items[u]=o,j(d,a),y({ok:!0,item:o,removed:l-o.tags.length,message:`Removed tag from ${o.id}`},n.json)});return}if(r==="upsert"){let a=n.title??t[1],u=n.content??t[2];D(d,()=>{let o=P(d),l=n.id?o.items.findIndex((E)=>E.id===n.id||E.short_id===n.id):-1,f=new Date().toISOString();if(l===-1){if(!a||!u)throw Error("New item requires title and content. Example: open-knowledge upsert <title> <content> [--id <id>]");let E=n.id??He(),h={id:E,short_id:kt(E),title:a,content:u,url:n.url??null,tags:n.tag?[n.tag]:[],metadata:{},archived:!1,created_at:f,updated_at:f};o.items.push(h),j(d,o),y({ok:!0,created:!0,item:h,message:`Upserted ${h.id}`},n.json);return}let g=o.items[l];if(a!==void 0)g.title=a;if(u!==void 0)g.content=u;if(n.url!==void 0)g.url=n.url;if(n.tag!==void 0){if(g.tags=g.tags||[],!g.tags.map((E)=>E.toLowerCase()).includes(n.tag.toLowerCase()))g.tags.push(n.tag)}g.updated_at=f,o.items[l]=g,j(d,o),y({ok:!0,created:!1,item:g,message:`Upserted ${g.id}`},n.json)});return}if(r==="delete"){if(_e(n),!n.yes)throw Error("Refusing delete without --yes. Re-run with: open-knowledge delete --id <id> --yes");D(d,()=>{let a=P(d),u=a.items.length;a.items=a.items.filter((l)=>l.id!==n.id&&l.short_id!==n.id);let o=u!==a.items.length;if(j(d,a),!o)throw Error(`Item not found: ${n.id}`);Q("info","Item deleted",{id:n.id}),y({ok:!0,deleted_id:n.id,message:`Deleted ${n.id}`},n.json)});return}if(r==="export"){let a=n.format??"json";if(a!=="json"&&a!=="jsonl")throw Error("Invalid --format. Use 'json' or 'jsonl'.");D(d,()=>{let u=P(d);if(a==="jsonl")for(let o of u.items)console.log(JSON.stringify(o));else y({ok:!0,items:u.items},n.json)});return}if(r==="prune"){if(!n.yes)throw Error("Refusing prune without --yes. Re-run with: open-knowledge prune --yes [--older-than <days>] [--empty]");D(d,()=>{let a=P(d),u=a.items.length;if(n.olderThan!==void 0){let l=new Date;l.setDate(l.getDate()-n.olderThan),a.items=a.items.filter((f)=>new Date(f.created_at)>=l)}if(n.empty)a.items=a.items.filter((l)=>l.content.trim().length>0);let o=u-a.items.length;j(d,a),Q("info","Prune completed",{pruned:o,remaining:a.items.length}),y({ok:!0,pruned:o,remaining:a.items.length,message:`Pruned ${o} item(s)`},n.json)});return}if(r==="dedupe"){if(!n.yes)throw Error("Refusing dedupe without --yes. Re-run with: open-knowledge dedupe --yes [--json]");D(d,()=>{let a=P(d),u=new Set,o=a.items.length;a.items=a.items.filter((f)=>{let g=`${f.title}\x00${f.content}`;if(u.has(g))return!1;return u.add(g),!0});let l=o-a.items.length;j(d,a),Q("info","Dedupe completed",{removed:l,remaining:a.items.length}),y({ok:!0,removed:l,remaining:a.items.length,message:`Dedupe removed ${l} duplicate(s)`},n.json)});return}if(r==="stats"){D(d,()=>{let a=P(d),u=a.items.filter((k)=>!k.archived),o=u.length,l=a.items.length-o,f=u.filter((k)=>k.url).length,g=u.filter((k)=>k.tags&&k.tags.length>0).length,E=o>0?u.map((k)=>k.created_at).sort()[0]:null,h=o>0?u.map((k)=>k.created_at).sort()[o-1]:null,m={};for(let k of u)for(let p of k.tags||[])m[p]=(m[p]||0)+1;let T=Object.entries(m).sort((k,p)=>p[1]-k[1]).slice(0,5).map(([k,p])=>({tag:k,count:p}));y({ok:!0,total:o,archived:l,with_url:f,with_tags:g,oldest:E,newest:h,top_tags:T,message:`${o} items | ${f} with URL | ${g} with tags`},n.json)});return}let c=Gs(t[0]),_=c?` Did you mean '${c}'?`:"";throw Q("warn","Unknown command",{input:t[0],suggestion:c}),Error(`Unknown command: ${t[0]}.${_} Run 'open-knowledge --help' for available commands.`)}if(import.meta.main)to(process.argv.slice(2)).catch((e)=>{let t=e instanceof Error?e.message:String(e);Q("error","CLI error",{message:t,stack:e instanceof Error?e.stack:void 0}),console.error(`Error: ${t}`),process.exitCode=1});export{Gs as suggestCommand,eo as sortItems,to as run,Bs as parseArgs};
|