@hasna/knowledge 0.2.20 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/bin/open-knowledge-mcp.js +927 -24
- package/bin/open-knowledge.js +153 -71
- package/docs/architecture/ai-native-knowledge-base.md +16 -0
- package/package.json +1 -1
- package/src/auth.ts +123 -0
- package/src/cli.ts +153 -10
- package/src/remote-client.ts +268 -0
- package/src/service.ts +142 -0
- package/src/storage-contract.ts +28 -0
- package/src/wiki-compiler.ts +711 -0
- package/src/workspace.ts +11 -0
|
@@ -13656,11 +13656,11 @@ function date4(params) {
|
|
|
13656
13656
|
// node_modules/zod/v4/classic/external.js
|
|
13657
13657
|
config(en_default());
|
|
13658
13658
|
// src/mcp.js
|
|
13659
|
-
import { existsSync as
|
|
13659
|
+
import { existsSync as existsSync8, readFileSync as readFileSync8, writeFileSync as writeFileSync5 } from "fs";
|
|
13660
13660
|
// package.json
|
|
13661
13661
|
var package_default = {
|
|
13662
13662
|
name: "@hasna/knowledge",
|
|
13663
|
-
version: "0.2.
|
|
13663
|
+
version: "0.2.22",
|
|
13664
13664
|
description: "Agent-friendly local knowledge CLI with JSON output, pagination, and safe destructive actions",
|
|
13665
13665
|
type: "module",
|
|
13666
13666
|
bin: {
|
|
@@ -13761,6 +13761,9 @@ function defaultKnowledgeConfig() {
|
|
|
13761
13761
|
return {
|
|
13762
13762
|
version: 1,
|
|
13763
13763
|
mode: "local",
|
|
13764
|
+
hosted: {
|
|
13765
|
+
api_url: "https://knowledge.hasna.xyz"
|
|
13766
|
+
},
|
|
13764
13767
|
storage: {
|
|
13765
13768
|
type: "local",
|
|
13766
13769
|
artifacts_root: "artifacts"
|
|
@@ -13846,6 +13849,11 @@ function readKnowledgeConfig(path) {
|
|
|
13846
13849
|
const raw = readFileSync(path, "utf8");
|
|
13847
13850
|
return JSON.parse(raw);
|
|
13848
13851
|
}
|
|
13852
|
+
function writeKnowledgeConfig(path, config2) {
|
|
13853
|
+
ensureParentDir(path);
|
|
13854
|
+
writeFileSync(path, `${JSON.stringify(config2, null, 2)}
|
|
13855
|
+
`);
|
|
13856
|
+
}
|
|
13849
13857
|
|
|
13850
13858
|
// src/store.ts
|
|
13851
13859
|
function defaultStorePath() {
|
|
@@ -14135,6 +14143,91 @@ function createArtifactStore(config2, workspace) {
|
|
|
14135
14143
|
return new LocalArtifactStore(workspace.artifactsDir);
|
|
14136
14144
|
}
|
|
14137
14145
|
|
|
14146
|
+
// src/auth.ts
|
|
14147
|
+
import { existsSync as existsSync4, mkdirSync as mkdirSync3, readFileSync as readFileSync4, unlinkSync as unlinkSync2, writeFileSync as writeFileSync4 } from "fs";
|
|
14148
|
+
import { homedir as homedir2 } from "os";
|
|
14149
|
+
import { dirname as dirname3, join as join3 } from "path";
|
|
14150
|
+
var DEFAULT_KNOWLEDGE_API_URL = "https://knowledge.hasna.xyz";
|
|
14151
|
+
function normalizeKnowledgeApiOrigin(apiUrl) {
|
|
14152
|
+
const url2 = new URL(apiUrl);
|
|
14153
|
+
if (url2.protocol !== "http:" && url2.protocol !== "https:") {
|
|
14154
|
+
throw new Error("Knowledge API URL must use http or https.");
|
|
14155
|
+
}
|
|
14156
|
+
const pathname = url2.pathname.replace(/\/+$/, "");
|
|
14157
|
+
if (pathname === "/api" || pathname === "/api/v1") {
|
|
14158
|
+
url2.pathname = "/";
|
|
14159
|
+
} else if (pathname.endsWith("/api/v1")) {
|
|
14160
|
+
url2.pathname = pathname.slice(0, -"/api/v1".length) || "/";
|
|
14161
|
+
} else if (pathname.endsWith("/api")) {
|
|
14162
|
+
url2.pathname = pathname.slice(0, -"/api".length) || "/";
|
|
14163
|
+
}
|
|
14164
|
+
return url2.toString().replace(/\/+$/, "");
|
|
14165
|
+
}
|
|
14166
|
+
function knowledgeAuthPath(env = process.env) {
|
|
14167
|
+
if (env.HASNA_KNOWLEDGE_AUTH_PATH)
|
|
14168
|
+
return env.HASNA_KNOWLEDGE_AUTH_PATH;
|
|
14169
|
+
const root = env.HASNA_KNOWLEDGE_AUTH_DIR ?? join3(homedir2(), ".hasna", "knowledge");
|
|
14170
|
+
return join3(root, "auth.json");
|
|
14171
|
+
}
|
|
14172
|
+
function resolveKnowledgeApiUrl(config2, env = process.env) {
|
|
14173
|
+
return normalizeKnowledgeApiOrigin(env.KNOWLEDGE_API_URL ?? config2?.hosted?.api_url ?? DEFAULT_KNOWLEDGE_API_URL);
|
|
14174
|
+
}
|
|
14175
|
+
function getKnowledgeAuth(env = process.env) {
|
|
14176
|
+
try {
|
|
14177
|
+
const path = knowledgeAuthPath(env);
|
|
14178
|
+
if (!existsSync4(path))
|
|
14179
|
+
return null;
|
|
14180
|
+
const parsed = JSON.parse(readFileSync4(path, "utf8"));
|
|
14181
|
+
return typeof parsed.api_key === "string" && parsed.api_key.length > 0 ? parsed : null;
|
|
14182
|
+
} catch {
|
|
14183
|
+
return null;
|
|
14184
|
+
}
|
|
14185
|
+
}
|
|
14186
|
+
function saveKnowledgeAuth(auth, env = process.env) {
|
|
14187
|
+
const path = knowledgeAuthPath(env);
|
|
14188
|
+
const stored = {
|
|
14189
|
+
...auth,
|
|
14190
|
+
api_url: auth.api_url ? normalizeKnowledgeApiOrigin(auth.api_url) : undefined,
|
|
14191
|
+
created_at: auth.created_at ?? new Date().toISOString()
|
|
14192
|
+
};
|
|
14193
|
+
mkdirSync3(dirname3(path), { recursive: true, mode: 448 });
|
|
14194
|
+
writeFileSync4(path, `${JSON.stringify(stored, null, 2)}
|
|
14195
|
+
`, { mode: 384 });
|
|
14196
|
+
return stored;
|
|
14197
|
+
}
|
|
14198
|
+
function clearKnowledgeAuth(env = process.env) {
|
|
14199
|
+
try {
|
|
14200
|
+
unlinkSync2(knowledgeAuthPath(env));
|
|
14201
|
+
return true;
|
|
14202
|
+
} catch {
|
|
14203
|
+
return false;
|
|
14204
|
+
}
|
|
14205
|
+
}
|
|
14206
|
+
function getKnowledgeApiKey(env = process.env) {
|
|
14207
|
+
if (env.KNOWLEDGE_API_KEY)
|
|
14208
|
+
return { apiKey: env.KNOWLEDGE_API_KEY, source: "env" };
|
|
14209
|
+
if (env.HASNA_KNOWLEDGE_API_KEY)
|
|
14210
|
+
return { apiKey: env.HASNA_KNOWLEDGE_API_KEY, source: "env" };
|
|
14211
|
+
const auth = getKnowledgeAuth(env);
|
|
14212
|
+
return auth?.api_key ? { apiKey: auth.api_key, source: "file" } : { apiKey: null, source: "none" };
|
|
14213
|
+
}
|
|
14214
|
+
function knowledgeAuthStatus(config2, env = process.env) {
|
|
14215
|
+
const auth = getKnowledgeAuth(env);
|
|
14216
|
+
const key = getKnowledgeApiKey(env);
|
|
14217
|
+
const apiUrl = env.KNOWLEDGE_API_URL ? resolveKnowledgeApiUrl(config2, env) : auth?.api_url ? normalizeKnowledgeApiOrigin(auth.api_url) : resolveKnowledgeApiUrl(config2, env);
|
|
14218
|
+
return {
|
|
14219
|
+
authenticated: Boolean(key.apiKey),
|
|
14220
|
+
source: key.source,
|
|
14221
|
+
api_url: apiUrl,
|
|
14222
|
+
auth_path: knowledgeAuthPath(env),
|
|
14223
|
+
email: key.source === "file" ? auth?.email ?? null : null,
|
|
14224
|
+
org_id: key.source === "file" ? auth?.org_id ?? null : null,
|
|
14225
|
+
org_slug: key.source === "file" ? auth?.org_slug ?? null : null,
|
|
14226
|
+
user_id: key.source === "file" ? auth?.user_id ?? null : null,
|
|
14227
|
+
api_key_present: Boolean(key.apiKey)
|
|
14228
|
+
};
|
|
14229
|
+
}
|
|
14230
|
+
|
|
14138
14231
|
// src/agent.ts
|
|
14139
14232
|
import { randomUUID as randomUUID3 } from "crypto";
|
|
14140
14233
|
|
|
@@ -15930,7 +16023,7 @@ ${answer}`;
|
|
|
15930
16023
|
|
|
15931
16024
|
// src/outbox-consume.ts
|
|
15932
16025
|
import { createHash as createHash4, randomUUID as randomUUID5 } from "crypto";
|
|
15933
|
-
import { existsSync as
|
|
16026
|
+
import { existsSync as existsSync5, readFileSync as readFileSync5 } from "fs";
|
|
15934
16027
|
import { basename } from "path";
|
|
15935
16028
|
|
|
15936
16029
|
// src/safety.ts
|
|
@@ -16203,9 +16296,9 @@ async function readS3Text(uri, config2, safetyPolicy) {
|
|
|
16203
16296
|
async function readOutboxInput(input, config2, safetyPolicy) {
|
|
16204
16297
|
if (input.startsWith("s3://"))
|
|
16205
16298
|
return readS3Text(input, config2, safetyPolicy);
|
|
16206
|
-
if (!
|
|
16299
|
+
if (!existsSync5(input))
|
|
16207
16300
|
throw new Error(`Outbox not found: ${input}`);
|
|
16208
|
-
return
|
|
16301
|
+
return readFileSync5(input, "utf8");
|
|
16209
16302
|
}
|
|
16210
16303
|
function mergeJson(existing, patch) {
|
|
16211
16304
|
let base = {};
|
|
@@ -16439,7 +16532,7 @@ async function consumeOpenFilesOutbox(options) {
|
|
|
16439
16532
|
|
|
16440
16533
|
// src/manifest-ingest.ts
|
|
16441
16534
|
import { createHash as createHash5 } from "crypto";
|
|
16442
|
-
import { existsSync as
|
|
16535
|
+
import { existsSync as existsSync6, readFileSync as readFileSync6 } from "fs";
|
|
16443
16536
|
import { basename as basename2 } from "path";
|
|
16444
16537
|
function stableId4(prefix, value) {
|
|
16445
16538
|
return `${prefix}_${createHash5("sha256").update(value).digest("hex").slice(0, 20)}`;
|
|
@@ -16611,9 +16704,9 @@ async function readS3Text2(uri, config2, safetyPolicy) {
|
|
|
16611
16704
|
async function readManifestInput(input, config2, safetyPolicy) {
|
|
16612
16705
|
if (input.startsWith("s3://"))
|
|
16613
16706
|
return readS3Text2(input, config2, safetyPolicy);
|
|
16614
|
-
if (!
|
|
16707
|
+
if (!existsSync6(input))
|
|
16615
16708
|
throw new Error(`Manifest not found: ${input}`);
|
|
16616
|
-
return
|
|
16709
|
+
return readFileSync6(input, "utf8");
|
|
16617
16710
|
}
|
|
16618
16711
|
function chunkText(text, maxChars, overlapChars) {
|
|
16619
16712
|
const normalized = text.replace(/\r\n/g, `
|
|
@@ -16857,7 +16950,7 @@ async function ingestOpenFilesManifestItems(options) {
|
|
|
16857
16950
|
|
|
16858
16951
|
// src/source-ingest.ts
|
|
16859
16952
|
import { createHash as createHash6 } from "crypto";
|
|
16860
|
-
import { existsSync as
|
|
16953
|
+
import { existsSync as existsSync7, readFileSync as readFileSync7 } from "fs";
|
|
16861
16954
|
import { basename as basename3 } from "path";
|
|
16862
16955
|
|
|
16863
16956
|
// src/source-resolver.ts
|
|
@@ -17207,9 +17300,9 @@ function titleForRef(parsed) {
|
|
|
17207
17300
|
}
|
|
17208
17301
|
async function readDirectSourceText(parsed, config2, safetyPolicy) {
|
|
17209
17302
|
if (parsed.kind === "file") {
|
|
17210
|
-
if (!
|
|
17303
|
+
if (!existsSync7(parsed.path))
|
|
17211
17304
|
throw new Error(`Source file not found: ${parsed.path}`);
|
|
17212
|
-
const text =
|
|
17305
|
+
const text = readFileSync7(parsed.path, "utf8");
|
|
17213
17306
|
return {
|
|
17214
17307
|
text,
|
|
17215
17308
|
contentSource: "file",
|
|
@@ -17560,6 +17653,164 @@ async function refreshEmbeddingIndex(options) {
|
|
|
17560
17653
|
};
|
|
17561
17654
|
}
|
|
17562
17655
|
|
|
17656
|
+
// src/remote-client.ts
|
|
17657
|
+
var REMOTE_KNOWLEDGE_CONTRACT_VERSION = 1;
|
|
17658
|
+
function isRecord(value) {
|
|
17659
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
17660
|
+
}
|
|
17661
|
+
function stringValue(record2, key) {
|
|
17662
|
+
const value = record2[key];
|
|
17663
|
+
return typeof value === "string" ? value : undefined;
|
|
17664
|
+
}
|
|
17665
|
+
function numberValue(record2, key) {
|
|
17666
|
+
const value = record2[key];
|
|
17667
|
+
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
|
17668
|
+
}
|
|
17669
|
+
function arrayValue(record2, key) {
|
|
17670
|
+
const value = record2[key];
|
|
17671
|
+
return Array.isArray(value) ? value : undefined;
|
|
17672
|
+
}
|
|
17673
|
+
function normalizeRemoteKnowledgeRunContract(payload, fallback) {
|
|
17674
|
+
const record2 = isRecord(payload) ? payload : {};
|
|
17675
|
+
return {
|
|
17676
|
+
contract_version: REMOTE_KNOWLEDGE_CONTRACT_VERSION,
|
|
17677
|
+
id: stringValue(record2, "id") ?? fallback?.id,
|
|
17678
|
+
type: stringValue(record2, "type") ?? fallback?.type,
|
|
17679
|
+
status: stringValue(record2, "status") ?? fallback?.status,
|
|
17680
|
+
query: stringValue(record2, "query") ?? fallback?.query,
|
|
17681
|
+
prompt: stringValue(record2, "prompt") ?? fallback?.prompt,
|
|
17682
|
+
output_preview: Object.prototype.hasOwnProperty.call(record2, "output_preview") ? record2.output_preview : fallback?.output_preview,
|
|
17683
|
+
citations: arrayValue(record2, "citations") ?? fallback?.citations,
|
|
17684
|
+
artifacts: arrayValue(record2, "artifacts") ?? fallback?.artifacts,
|
|
17685
|
+
usage: isRecord(record2.usage) ? record2.usage : fallback?.usage,
|
|
17686
|
+
created_at: stringValue(record2, "created_at") ?? fallback?.created_at,
|
|
17687
|
+
started_at: stringValue(record2, "started_at") ?? fallback?.started_at,
|
|
17688
|
+
completed_at: stringValue(record2, "completed_at") ?? fallback?.completed_at,
|
|
17689
|
+
duration_ms: numberValue(record2, "duration_ms") ?? fallback?.duration_ms,
|
|
17690
|
+
error_code: stringValue(record2, "error_code") ?? fallback?.error_code,
|
|
17691
|
+
error_message: stringValue(record2, "error_message") ?? fallback?.error_message,
|
|
17692
|
+
error: stringValue(record2, "error") ?? fallback?.error,
|
|
17693
|
+
details: Object.prototype.hasOwnProperty.call(record2, "details") ? record2.details : fallback?.details
|
|
17694
|
+
};
|
|
17695
|
+
}
|
|
17696
|
+
function knowledgeRegistryContract(input) {
|
|
17697
|
+
return {
|
|
17698
|
+
contract_version: REMOTE_KNOWLEDGE_CONTRACT_VERSION,
|
|
17699
|
+
service: "open-knowledge",
|
|
17700
|
+
mode: input.mode,
|
|
17701
|
+
capabilities: [
|
|
17702
|
+
"registry",
|
|
17703
|
+
"search",
|
|
17704
|
+
"ask",
|
|
17705
|
+
"build",
|
|
17706
|
+
"sync",
|
|
17707
|
+
"status",
|
|
17708
|
+
"logs",
|
|
17709
|
+
"artifacts",
|
|
17710
|
+
"open-files-source-refs",
|
|
17711
|
+
"s3-generated-artifacts"
|
|
17712
|
+
],
|
|
17713
|
+
endpoints: {
|
|
17714
|
+
registry: "/api/v1/knowledge/registry",
|
|
17715
|
+
search: "/api/v1/knowledge/search",
|
|
17716
|
+
ask: "/api/v1/knowledge/ask",
|
|
17717
|
+
build: "/api/v1/knowledge/build",
|
|
17718
|
+
sync: "/api/v1/knowledge/sync",
|
|
17719
|
+
run_status: "/api/v1/knowledge/runs/{run_id}",
|
|
17720
|
+
run_logs: "/api/v1/knowledge/runs/{run_id}/logs",
|
|
17721
|
+
run_artifacts: "/api/v1/knowledge/runs/{run_id}/artifacts"
|
|
17722
|
+
},
|
|
17723
|
+
source_contract: {
|
|
17724
|
+
owner: "open-files",
|
|
17725
|
+
preferred_ref: "open-files",
|
|
17726
|
+
allowed_schemes: input.sourceSchemes,
|
|
17727
|
+
raw_source_bytes_stored_in_open_knowledge: false
|
|
17728
|
+
},
|
|
17729
|
+
artifact_contract: {
|
|
17730
|
+
storage_type: input.storageType,
|
|
17731
|
+
uri_prefix: input.artifactUriPrefix,
|
|
17732
|
+
generated_only: true
|
|
17733
|
+
}
|
|
17734
|
+
};
|
|
17735
|
+
}
|
|
17736
|
+
|
|
17737
|
+
class RemoteKnowledgeClient {
|
|
17738
|
+
apiKey;
|
|
17739
|
+
apiUrl;
|
|
17740
|
+
constructor(apiKey, apiUrl) {
|
|
17741
|
+
this.apiKey = apiKey;
|
|
17742
|
+
this.apiUrl = apiUrl;
|
|
17743
|
+
}
|
|
17744
|
+
static fromConfig(config2, env = process.env) {
|
|
17745
|
+
const key = getKnowledgeApiKey(env);
|
|
17746
|
+
if (!key.apiKey)
|
|
17747
|
+
return null;
|
|
17748
|
+
return new RemoteKnowledgeClient(key.apiKey, resolveKnowledgeApiUrl(config2, env));
|
|
17749
|
+
}
|
|
17750
|
+
async request(path, options = {}) {
|
|
17751
|
+
return fetch(`${this.apiUrl}${path}`, {
|
|
17752
|
+
...options,
|
|
17753
|
+
headers: {
|
|
17754
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
17755
|
+
"Content-Type": "application/json",
|
|
17756
|
+
...options.headers
|
|
17757
|
+
}
|
|
17758
|
+
});
|
|
17759
|
+
}
|
|
17760
|
+
async registry() {
|
|
17761
|
+
const response = await this.request("/api/v1/knowledge/registry");
|
|
17762
|
+
return response.json();
|
|
17763
|
+
}
|
|
17764
|
+
async search(request) {
|
|
17765
|
+
const response = await this.request("/api/v1/knowledge/search", {
|
|
17766
|
+
method: "POST",
|
|
17767
|
+
body: JSON.stringify(request)
|
|
17768
|
+
});
|
|
17769
|
+
return normalizeRemoteKnowledgeRunContract(await response.json(), { type: "search", query: request.query });
|
|
17770
|
+
}
|
|
17771
|
+
async ask(request) {
|
|
17772
|
+
const response = await this.request("/api/v1/knowledge/ask", {
|
|
17773
|
+
method: "POST",
|
|
17774
|
+
body: JSON.stringify(request)
|
|
17775
|
+
});
|
|
17776
|
+
return normalizeRemoteKnowledgeRunContract(await response.json(), { type: "ask", prompt: request.prompt });
|
|
17777
|
+
}
|
|
17778
|
+
async build(request) {
|
|
17779
|
+
const response = await this.request("/api/v1/knowledge/build", {
|
|
17780
|
+
method: "POST",
|
|
17781
|
+
body: JSON.stringify(request)
|
|
17782
|
+
});
|
|
17783
|
+
return normalizeRemoteKnowledgeRunContract(await response.json(), { type: "build", prompt: request.prompt });
|
|
17784
|
+
}
|
|
17785
|
+
async sync(request = {}) {
|
|
17786
|
+
const response = await this.request("/api/v1/knowledge/sync", {
|
|
17787
|
+
method: "POST",
|
|
17788
|
+
body: JSON.stringify(request)
|
|
17789
|
+
});
|
|
17790
|
+
return normalizeRemoteKnowledgeRunContract(await response.json(), { type: "sync" });
|
|
17791
|
+
}
|
|
17792
|
+
async runStatus(runId) {
|
|
17793
|
+
const response = await this.request(`/api/v1/knowledge/runs/${encodeURIComponent(runId)}`);
|
|
17794
|
+
if (!response.ok)
|
|
17795
|
+
return null;
|
|
17796
|
+
return normalizeRemoteKnowledgeRunContract(await response.json(), { id: runId, type: "status" });
|
|
17797
|
+
}
|
|
17798
|
+
async runLogs(runId) {
|
|
17799
|
+
const response = await this.request(`/api/v1/knowledge/runs/${encodeURIComponent(runId)}/logs`);
|
|
17800
|
+
if (!response.ok)
|
|
17801
|
+
return [];
|
|
17802
|
+
const payload = await response.json();
|
|
17803
|
+
return Array.isArray(payload) ? payload : [];
|
|
17804
|
+
}
|
|
17805
|
+
async runArtifacts(runId) {
|
|
17806
|
+
const response = await this.request(`/api/v1/knowledge/runs/${encodeURIComponent(runId)}/artifacts`);
|
|
17807
|
+
if (!response.ok)
|
|
17808
|
+
return [];
|
|
17809
|
+
const payload = await response.json();
|
|
17810
|
+
return Array.isArray(payload) ? payload : [];
|
|
17811
|
+
}
|
|
17812
|
+
}
|
|
17813
|
+
|
|
17563
17814
|
// src/web-search.ts
|
|
17564
17815
|
import { createHash as createHash8, randomUUID as randomUUID7 } from "crypto";
|
|
17565
17816
|
function stableHash(value) {
|
|
@@ -17811,6 +18062,9 @@ async function runProviderWebSearch(options) {
|
|
|
17811
18062
|
};
|
|
17812
18063
|
}
|
|
17813
18064
|
|
|
18065
|
+
// src/wiki-compiler.ts
|
|
18066
|
+
import { createHash as createHash10, randomUUID as randomUUID9 } from "crypto";
|
|
18067
|
+
|
|
17814
18068
|
// src/storage-contract.ts
|
|
17815
18069
|
import { createHash as createHash9, randomUUID as randomUUID8 } from "crypto";
|
|
17816
18070
|
var GENERATED_ARTIFACTS = [
|
|
@@ -17895,6 +18149,15 @@ function resolveStorageContract(config2, workspace, scope = "global") {
|
|
|
17895
18149
|
kms_key_configured: Boolean(s3.kms_key_id)
|
|
17896
18150
|
} : null
|
|
17897
18151
|
},
|
|
18152
|
+
hosted: {
|
|
18153
|
+
enabled: config2.mode === "hosted",
|
|
18154
|
+
api_url: normalizeKnowledgeApiOrigin(config2.hosted?.api_url ?? DEFAULT_KNOWLEDGE_API_URL),
|
|
18155
|
+
api_url_env: "KNOWLEDGE_API_URL",
|
|
18156
|
+
api_key_env: "KNOWLEDGE_API_KEY",
|
|
18157
|
+
auth_storage: "~/.hasna/knowledge/auth.json",
|
|
18158
|
+
remote_contract_version: REMOTE_KNOWLEDGE_CONTRACT_VERSION,
|
|
18159
|
+
requires_hosted_account_for_local_use: false
|
|
18160
|
+
},
|
|
17898
18161
|
source_ownership: {
|
|
17899
18162
|
owner: "open-files",
|
|
17900
18163
|
preferred_ref: config2.sources.preferred_ref,
|
|
@@ -17950,6 +18213,13 @@ function validateStorageConfig(config2, workspace) {
|
|
|
17950
18213
|
if (!config2.sources.allowed_schemes.includes("open-files")) {
|
|
17951
18214
|
errors3.push("sources.allowed_schemes must include open-files.");
|
|
17952
18215
|
}
|
|
18216
|
+
if (config2.mode === "hosted" && config2.hosted?.api_url) {
|
|
18217
|
+
try {
|
|
18218
|
+
normalizeKnowledgeApiOrigin(config2.hosted.api_url);
|
|
18219
|
+
} catch {
|
|
18220
|
+
errors3.push("hosted.api_url must be an http(s) URL when mode is hosted.");
|
|
18221
|
+
}
|
|
18222
|
+
}
|
|
17953
18223
|
return {
|
|
17954
18224
|
ok: errors3.length === 0,
|
|
17955
18225
|
errors: errors3,
|
|
@@ -17982,18 +18252,556 @@ function recordStorageObjects(db, objects, now = new Date) {
|
|
|
17982
18252
|
insert(objects);
|
|
17983
18253
|
}
|
|
17984
18254
|
|
|
17985
|
-
// src/wiki-
|
|
17986
|
-
|
|
18255
|
+
// src/wiki-compiler.ts
|
|
18256
|
+
function stableId6(prefix, value) {
|
|
18257
|
+
return `${prefix}_${createHash10("sha256").update(value).digest("hex").slice(0, 20)}`;
|
|
18258
|
+
}
|
|
18259
|
+
function slugify2(value) {
|
|
18260
|
+
const slug = value.normalize("NFKC").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
18261
|
+
return slug || "knowledge-page";
|
|
18262
|
+
}
|
|
17987
18263
|
function todayParts(now) {
|
|
18264
|
+
return {
|
|
18265
|
+
year: String(now.getUTCFullYear()),
|
|
18266
|
+
month: String(now.getUTCMonth() + 1).padStart(2, "0"),
|
|
18267
|
+
day: String(now.getUTCDate()).padStart(2, "0")
|
|
18268
|
+
};
|
|
18269
|
+
}
|
|
18270
|
+
function estimateTokenCount2(text) {
|
|
18271
|
+
const words = text.trim().split(/\s+/).filter(Boolean).length;
|
|
18272
|
+
return Math.max(1, Math.ceil(words * 1.25));
|
|
18273
|
+
}
|
|
18274
|
+
function parseJsonObject4(value) {
|
|
18275
|
+
if (!value)
|
|
18276
|
+
return {};
|
|
18277
|
+
try {
|
|
18278
|
+
const parsed = JSON.parse(value);
|
|
18279
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
|
|
18280
|
+
} catch {
|
|
18281
|
+
return {};
|
|
18282
|
+
}
|
|
18283
|
+
}
|
|
18284
|
+
function queryTerms3(query) {
|
|
18285
|
+
return Array.from(new Set((query ?? "").toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [])).slice(0, 12);
|
|
18286
|
+
}
|
|
18287
|
+
function escapeLike(value) {
|
|
18288
|
+
return value.replace(/[\\%_]/g, (char) => `\\${char}`);
|
|
18289
|
+
}
|
|
18290
|
+
function selectSourceChunks(db, options) {
|
|
18291
|
+
const limit = Math.max(1, Math.min(options.limit ?? 10, 50));
|
|
18292
|
+
const sourceRefs = options.sourceRefs ?? [];
|
|
18293
|
+
const terms = queryTerms3(options.query);
|
|
18294
|
+
const where = ["c.kind = 'source'"];
|
|
18295
|
+
const params = [];
|
|
18296
|
+
if (sourceRefs.length > 0) {
|
|
18297
|
+
where.push(`(${sourceRefs.map(() => "(s.uri = ? OR c.metadata_json LIKE ?)").join(" OR ")})`);
|
|
18298
|
+
for (const ref of sourceRefs) {
|
|
18299
|
+
params.push(ref, `%${escapeLike(ref)}%`);
|
|
18300
|
+
}
|
|
18301
|
+
}
|
|
18302
|
+
if (terms.length > 0) {
|
|
18303
|
+
where.push(`(${terms.map(() => "lower(c.text) LIKE ? ESCAPE '\\'").join(" OR ")})`);
|
|
18304
|
+
for (const term of terms)
|
|
18305
|
+
params.push(`%${escapeLike(term)}%`);
|
|
18306
|
+
}
|
|
18307
|
+
params.push(limit);
|
|
18308
|
+
return db.query(`SELECT
|
|
18309
|
+
c.id AS chunk_id,
|
|
18310
|
+
c.text,
|
|
18311
|
+
c.start_offset,
|
|
18312
|
+
c.end_offset,
|
|
18313
|
+
c.metadata_json,
|
|
18314
|
+
c.source_revision_id,
|
|
18315
|
+
sr.revision,
|
|
18316
|
+
sr.hash,
|
|
18317
|
+
s.uri AS source_uri,
|
|
18318
|
+
s.title AS source_title
|
|
18319
|
+
FROM chunks c
|
|
18320
|
+
JOIN source_revisions sr ON sr.id = c.source_revision_id
|
|
18321
|
+
JOIN sources s ON s.id = sr.source_id
|
|
18322
|
+
WHERE ${where.join(" AND ")}
|
|
18323
|
+
ORDER BY c.created_at ASC, c.ordinal ASC
|
|
18324
|
+
LIMIT ?`).all(...params);
|
|
18325
|
+
}
|
|
18326
|
+
function excerpt(text, max = 420) {
|
|
18327
|
+
const normalized = text.replace(/\s+/g, " ").trim();
|
|
18328
|
+
return normalized.length <= max ? normalized : `${normalized.slice(0, max - 1).trim()}...`;
|
|
18329
|
+
}
|
|
18330
|
+
function titleFor(options, rows) {
|
|
18331
|
+
if (options.title?.trim())
|
|
18332
|
+
return options.title.trim();
|
|
18333
|
+
if (options.query?.trim())
|
|
18334
|
+
return options.query.trim();
|
|
18335
|
+
return rows[0]?.source_title ?? "Compiled Knowledge";
|
|
18336
|
+
}
|
|
18337
|
+
function compileBody(title, rows, now) {
|
|
18338
|
+
const sourceLines = rows.map((row, index) => {
|
|
18339
|
+
const label = `S${index + 1}`;
|
|
18340
|
+
return `- [${label}] ${row.source_title ?? row.source_uri ?? "Source"} (${row.source_uri ?? "unknown"}, revision ${row.revision ?? "unknown"}, hash ${row.hash ?? "unknown"})`;
|
|
18341
|
+
});
|
|
18342
|
+
const noteLines = rows.map((row, index) => {
|
|
18343
|
+
const label = `S${index + 1}`;
|
|
18344
|
+
return [
|
|
18345
|
+
`## ${row.source_title ?? `Source ${index + 1}`}`,
|
|
18346
|
+
"",
|
|
18347
|
+
excerpt(row.text),
|
|
18348
|
+
"",
|
|
18349
|
+
`Citation: [${label}]`
|
|
18350
|
+
].join(`
|
|
18351
|
+
`);
|
|
18352
|
+
});
|
|
18353
|
+
return [
|
|
18354
|
+
`# ${title}`,
|
|
18355
|
+
"",
|
|
18356
|
+
`Generated at: ${now}`,
|
|
18357
|
+
"",
|
|
18358
|
+
"## Sources",
|
|
18359
|
+
"",
|
|
18360
|
+
...sourceLines,
|
|
18361
|
+
"",
|
|
18362
|
+
...noteLines,
|
|
18363
|
+
""
|
|
18364
|
+
].join(`
|
|
18365
|
+
`);
|
|
18366
|
+
}
|
|
18367
|
+
async function writeArtifact(store, entry) {
|
|
18368
|
+
const written = await store.put(entry);
|
|
18369
|
+
return {
|
|
18370
|
+
key: written.key,
|
|
18371
|
+
uri: written.uri,
|
|
18372
|
+
kind: entry.key.startsWith("logs/") ? "log" : "wiki_page",
|
|
18373
|
+
content_type: entry.content_type,
|
|
18374
|
+
...hashArtifactBody(entry.body),
|
|
18375
|
+
metadata: {
|
|
18376
|
+
...entry.metadata ?? {}
|
|
18377
|
+
}
|
|
18378
|
+
};
|
|
18379
|
+
}
|
|
18380
|
+
async function appendLog(store, event, now) {
|
|
18381
|
+
const { year, month, day } = todayParts(now);
|
|
18382
|
+
const key = `logs/${year}/${month}/${day}.jsonl`;
|
|
18383
|
+
let existing = "";
|
|
18384
|
+
try {
|
|
18385
|
+
existing = await store.getText(key);
|
|
18386
|
+
} catch {
|
|
18387
|
+
existing = "";
|
|
18388
|
+
}
|
|
18389
|
+
return writeArtifact(store, {
|
|
18390
|
+
key,
|
|
18391
|
+
body: `${existing}${JSON.stringify(event)}
|
|
18392
|
+
`,
|
|
18393
|
+
content_type: "application/x-ndjson"
|
|
18394
|
+
});
|
|
18395
|
+
}
|
|
18396
|
+
function upsertWikiPage(db, input) {
|
|
18397
|
+
db.run(`INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
|
|
18398
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
18399
|
+
ON CONFLICT(path) DO UPDATE SET
|
|
18400
|
+
title = excluded.title,
|
|
18401
|
+
artifact_uri = excluded.artifact_uri,
|
|
18402
|
+
content_hash = excluded.content_hash,
|
|
18403
|
+
status = excluded.status,
|
|
18404
|
+
metadata_json = excluded.metadata_json,
|
|
18405
|
+
updated_at = excluded.updated_at`, [
|
|
18406
|
+
input.pageId,
|
|
18407
|
+
input.path,
|
|
18408
|
+
input.title,
|
|
18409
|
+
input.artifactUri,
|
|
18410
|
+
input.contentHash,
|
|
18411
|
+
"active",
|
|
18412
|
+
JSON.stringify({
|
|
18413
|
+
artifact_key: input.path,
|
|
18414
|
+
provenance: input.provenance
|
|
18415
|
+
}),
|
|
18416
|
+
input.now,
|
|
18417
|
+
input.now
|
|
18418
|
+
]);
|
|
18419
|
+
const existing = db.query("SELECT id FROM chunks WHERE wiki_page_id = ?").all(input.pageId);
|
|
18420
|
+
for (const row of existing)
|
|
18421
|
+
db.run("DELETE FROM chunks_fts WHERE chunk_id = ?", [row.id]);
|
|
18422
|
+
db.run("DELETE FROM chunks WHERE wiki_page_id = ?", [input.pageId]);
|
|
18423
|
+
const chunkId = stableId6("chk", `${input.pageId}\x00${input.contentHash}`);
|
|
18424
|
+
db.run(`INSERT INTO chunks (id, wiki_page_id, kind, ordinal, text, token_count, start_offset, end_offset, metadata_json, created_at)
|
|
18425
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
18426
|
+
chunkId,
|
|
18427
|
+
input.pageId,
|
|
18428
|
+
"wiki",
|
|
18429
|
+
0,
|
|
18430
|
+
input.body,
|
|
18431
|
+
estimateTokenCount2(input.body),
|
|
18432
|
+
0,
|
|
18433
|
+
input.body.length,
|
|
18434
|
+
JSON.stringify({
|
|
18435
|
+
artifact_key: input.path,
|
|
18436
|
+
artifact_uri: input.artifactUri,
|
|
18437
|
+
content_hash: input.contentHash,
|
|
18438
|
+
provenance: input.provenance
|
|
18439
|
+
}),
|
|
18440
|
+
input.now
|
|
18441
|
+
]);
|
|
18442
|
+
db.run("INSERT INTO chunks_fts (chunk_id, text, title, source_uri) VALUES (?, ?, ?, ?)", [
|
|
18443
|
+
chunkId,
|
|
18444
|
+
input.body,
|
|
18445
|
+
input.title,
|
|
18446
|
+
input.artifactUri
|
|
18447
|
+
]);
|
|
18448
|
+
}
|
|
18449
|
+
function replacePageCitations(db, pageId, citations, now) {
|
|
18450
|
+
db.run("DELETE FROM citations WHERE wiki_page_id = ?", [pageId]);
|
|
18451
|
+
for (const citation of citations) {
|
|
18452
|
+
db.run(`INSERT INTO citations (id, wiki_page_id, chunk_id, source_uri, quote, start_offset, end_offset, metadata_json, created_at)
|
|
18453
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
18454
|
+
stableId6("cit", `${pageId}\x00${citation.source_uri}\x00${citation.chunk_id ?? randomUUID9()}`),
|
|
18455
|
+
pageId,
|
|
18456
|
+
citation.chunk_id,
|
|
18457
|
+
citation.source_uri,
|
|
18458
|
+
citation.quote,
|
|
18459
|
+
citation.start_offset,
|
|
18460
|
+
citation.end_offset,
|
|
18461
|
+
JSON.stringify(citation.metadata),
|
|
18462
|
+
now
|
|
18463
|
+
]);
|
|
18464
|
+
}
|
|
18465
|
+
return citations.length;
|
|
18466
|
+
}
|
|
18467
|
+
function upsertIndex(db, input) {
|
|
18468
|
+
db.run(`INSERT INTO knowledge_indexes (id, kind, name, artifact_uri, shard_key, metadata_json, created_at, updated_at)
|
|
18469
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
18470
|
+
ON CONFLICT(kind, name, shard_key) DO UPDATE SET
|
|
18471
|
+
artifact_uri = excluded.artifact_uri,
|
|
18472
|
+
metadata_json = excluded.metadata_json,
|
|
18473
|
+
updated_at = excluded.updated_at`, [
|
|
18474
|
+
stableId6("idx", `wiki-topic\x00${input.path}`),
|
|
18475
|
+
"wiki_topic",
|
|
18476
|
+
input.title,
|
|
18477
|
+
input.artifactUri,
|
|
18478
|
+
input.path,
|
|
18479
|
+
JSON.stringify({
|
|
18480
|
+
artifact_key: input.path,
|
|
18481
|
+
content_hash: input.contentHash
|
|
18482
|
+
}),
|
|
18483
|
+
input.now,
|
|
18484
|
+
input.now
|
|
18485
|
+
]);
|
|
18486
|
+
return 1;
|
|
18487
|
+
}
|
|
18488
|
+
function firstConcept(title) {
|
|
18489
|
+
return title.toLowerCase().match(/[a-z0-9][a-z0-9-]{2,}/)?.[0] ?? "knowledge";
|
|
18490
|
+
}
|
|
18491
|
+
async function compileWikiPage(options) {
|
|
18492
|
+
const nowDate = options.now ?? new Date;
|
|
18493
|
+
const now = nowDate.toISOString();
|
|
18494
|
+
migrateKnowledgeDb(options.dbPath);
|
|
18495
|
+
const readDb = openKnowledgeDb(options.dbPath);
|
|
18496
|
+
let rows;
|
|
18497
|
+
try {
|
|
18498
|
+
rows = selectSourceChunks(readDb, options);
|
|
18499
|
+
} finally {
|
|
18500
|
+
readDb.close();
|
|
18501
|
+
}
|
|
18502
|
+
if (rows.length === 0)
|
|
18503
|
+
throw new Error("No source chunks matched wiki compile input.");
|
|
18504
|
+
const title = titleFor(options, rows);
|
|
18505
|
+
const slug = slugify2(title);
|
|
18506
|
+
const path = `wiki/generated/${slug}.md`;
|
|
18507
|
+
const body = compileBody(title, rows, now);
|
|
18508
|
+
const sourceRefs = rows.map((row) => {
|
|
18509
|
+
const metadata = parseJsonObject4(row.metadata_json);
|
|
18510
|
+
return typeof metadata.source_ref === "string" ? metadata.source_ref : row.source_uri;
|
|
18511
|
+
}).filter((ref) => Boolean(ref));
|
|
18512
|
+
const provenance = generatedArtifactProvenance({
|
|
18513
|
+
generated_from: "wiki_compile",
|
|
18514
|
+
artifact_key: path,
|
|
18515
|
+
source_refs: sourceRefs
|
|
18516
|
+
});
|
|
18517
|
+
const pageArtifact = await writeArtifact(options.store, {
|
|
18518
|
+
key: path,
|
|
18519
|
+
body,
|
|
18520
|
+
content_type: "text/markdown",
|
|
18521
|
+
metadata: { generated_from: "wiki_compile" }
|
|
18522
|
+
});
|
|
18523
|
+
const pageId = stableId6("wiki", path);
|
|
18524
|
+
const citations = rows.map((row) => ({
|
|
18525
|
+
chunk_id: row.chunk_id,
|
|
18526
|
+
source_uri: row.source_uri ?? "unknown",
|
|
18527
|
+
quote: excerpt(row.text, 240),
|
|
18528
|
+
start_offset: row.start_offset,
|
|
18529
|
+
end_offset: row.end_offset,
|
|
18530
|
+
metadata: {
|
|
18531
|
+
source_revision_id: row.source_revision_id,
|
|
18532
|
+
revision: row.revision,
|
|
18533
|
+
hash: row.hash,
|
|
18534
|
+
source_ref: parseJsonObject4(row.metadata_json).source_ref ?? row.source_uri
|
|
18535
|
+
}
|
|
18536
|
+
}));
|
|
18537
|
+
const concept = firstConcept(title);
|
|
18538
|
+
const conceptPath = `wiki/concepts/${slugify2(concept)}.md`;
|
|
18539
|
+
const conceptBody = [`# ${concept}`, "", `Related page: [[${path}]]`, ""].join(`
|
|
18540
|
+
`);
|
|
18541
|
+
const conceptProvenance = generatedArtifactProvenance({
|
|
18542
|
+
generated_from: "wiki_compile_concept",
|
|
18543
|
+
artifact_key: conceptPath,
|
|
18544
|
+
source_refs: sourceRefs
|
|
18545
|
+
});
|
|
18546
|
+
const conceptArtifact = await writeArtifact(options.store, {
|
|
18547
|
+
key: conceptPath,
|
|
18548
|
+
body: conceptBody,
|
|
18549
|
+
content_type: "text/markdown",
|
|
18550
|
+
metadata: { generated_from: "wiki_compile_concept" }
|
|
18551
|
+
});
|
|
18552
|
+
const conceptPageId = stableId6("wiki", conceptPath);
|
|
18553
|
+
const log = await appendLog(options.store, {
|
|
18554
|
+
ts: now,
|
|
18555
|
+
event: "wiki_compile_completed",
|
|
18556
|
+
page_key: path,
|
|
18557
|
+
source_refs: sourceRefs,
|
|
18558
|
+
chunks_seen: rows.length
|
|
18559
|
+
}, nowDate);
|
|
18560
|
+
const db = openKnowledgeDb(options.dbPath);
|
|
18561
|
+
try {
|
|
18562
|
+
recordStorageObjects(db, [pageArtifact, conceptArtifact, log], nowDate);
|
|
18563
|
+
upsertWikiPage(db, {
|
|
18564
|
+
pageId,
|
|
18565
|
+
path,
|
|
18566
|
+
title,
|
|
18567
|
+
artifactUri: pageArtifact.uri,
|
|
18568
|
+
contentHash: pageArtifact.hash ?? "",
|
|
18569
|
+
body,
|
|
18570
|
+
provenance,
|
|
18571
|
+
now
|
|
18572
|
+
});
|
|
18573
|
+
upsertWikiPage(db, {
|
|
18574
|
+
pageId: conceptPageId,
|
|
18575
|
+
path: conceptPath,
|
|
18576
|
+
title: concept,
|
|
18577
|
+
artifactUri: conceptArtifact.uri,
|
|
18578
|
+
contentHash: conceptArtifact.hash ?? "",
|
|
18579
|
+
body: conceptBody,
|
|
18580
|
+
provenance: conceptProvenance,
|
|
18581
|
+
now
|
|
18582
|
+
});
|
|
18583
|
+
db.run(`INSERT OR REPLACE INTO wiki_backlinks (from_page_id, to_page_id, label, created_at)
|
|
18584
|
+
VALUES (?, ?, ?, ?)`, [pageId, conceptPageId, "concept", now]);
|
|
18585
|
+
const citationsWritten = replacePageCitations(db, pageId, citations, now);
|
|
18586
|
+
const indexesUpdated = upsertIndex(db, {
|
|
18587
|
+
title,
|
|
18588
|
+
path,
|
|
18589
|
+
artifactUri: pageArtifact.uri,
|
|
18590
|
+
contentHash: pageArtifact.hash ?? "",
|
|
18591
|
+
now
|
|
18592
|
+
});
|
|
18593
|
+
return {
|
|
18594
|
+
page_id: pageId,
|
|
18595
|
+
path,
|
|
18596
|
+
artifact_uri: pageArtifact.uri,
|
|
18597
|
+
content_hash: pageArtifact.hash ?? "",
|
|
18598
|
+
chunks_seen: rows.length,
|
|
18599
|
+
citations_written: citationsWritten,
|
|
18600
|
+
concept_page_id: conceptPageId,
|
|
18601
|
+
indexes_updated: indexesUpdated,
|
|
18602
|
+
log_key: log.key,
|
|
18603
|
+
warnings: []
|
|
18604
|
+
};
|
|
18605
|
+
} finally {
|
|
18606
|
+
db.close();
|
|
18607
|
+
}
|
|
18608
|
+
}
|
|
18609
|
+
async function fileAnswerToWiki(options) {
|
|
18610
|
+
if (!options.approveWrite) {
|
|
18611
|
+
return {
|
|
18612
|
+
approved: false,
|
|
18613
|
+
durable_writes_performed: false,
|
|
18614
|
+
page_id: null,
|
|
18615
|
+
path: null,
|
|
18616
|
+
artifact_uri: null,
|
|
18617
|
+
citations_written: 0,
|
|
18618
|
+
log_key: null,
|
|
18619
|
+
message: "Dry-run: answer filing requires --approve-write."
|
|
18620
|
+
};
|
|
18621
|
+
}
|
|
18622
|
+
const nowDate = options.now ?? new Date;
|
|
18623
|
+
const now = nowDate.toISOString();
|
|
18624
|
+
const title = options.prompt.length > 80 ? `${options.prompt.slice(0, 77)}...` : options.prompt;
|
|
18625
|
+
const slug = slugify2(title);
|
|
18626
|
+
const path = `wiki/answers/${slug}.md`;
|
|
18627
|
+
const citations = options.context.citations;
|
|
18628
|
+
const body = [
|
|
18629
|
+
`# ${title}`,
|
|
18630
|
+
"",
|
|
18631
|
+
options.answer,
|
|
18632
|
+
"",
|
|
18633
|
+
"## Citations",
|
|
18634
|
+
"",
|
|
18635
|
+
...citations.map((citation, index) => `- [C${index + 1}] ${citation.source_ref ?? citation.source_uri ?? citation.artifact_path ?? citation.artifact_uri ?? "unknown"} ${citation.hash ? `(hash ${citation.hash})` : ""}`),
|
|
18636
|
+
""
|
|
18637
|
+
].join(`
|
|
18638
|
+
`);
|
|
18639
|
+
const sourceRefs = citations.map((citation) => citation.source_ref ?? citation.source_uri).filter((ref) => Boolean(ref));
|
|
18640
|
+
const provenance = generatedArtifactProvenance({
|
|
18641
|
+
generated_from: "knowledge_answer",
|
|
18642
|
+
artifact_key: path,
|
|
18643
|
+
source_refs: sourceRefs
|
|
18644
|
+
});
|
|
18645
|
+
const artifact = await writeArtifact(options.store, {
|
|
18646
|
+
key: path,
|
|
18647
|
+
body,
|
|
18648
|
+
content_type: "text/markdown",
|
|
18649
|
+
metadata: { generated_from: "knowledge_answer" }
|
|
18650
|
+
});
|
|
18651
|
+
const log = await appendLog(options.store, {
|
|
18652
|
+
ts: now,
|
|
18653
|
+
event: "wiki_answer_filed",
|
|
18654
|
+
page_key: path,
|
|
18655
|
+
prompt: options.prompt,
|
|
18656
|
+
citations: citations.length
|
|
18657
|
+
}, nowDate);
|
|
18658
|
+
const pageId = stableId6("wiki", path);
|
|
18659
|
+
const db = openKnowledgeDb(options.dbPath);
|
|
18660
|
+
try {
|
|
18661
|
+
recordStorageObjects(db, [artifact, log], nowDate);
|
|
18662
|
+
upsertWikiPage(db, {
|
|
18663
|
+
pageId,
|
|
18664
|
+
path,
|
|
18665
|
+
title,
|
|
18666
|
+
artifactUri: artifact.uri,
|
|
18667
|
+
contentHash: artifact.hash ?? "",
|
|
18668
|
+
body,
|
|
18669
|
+
provenance,
|
|
18670
|
+
now
|
|
18671
|
+
});
|
|
18672
|
+
const written = replacePageCitations(db, pageId, citations.map((citation) => ({
|
|
18673
|
+
chunk_id: citation.chunk_id,
|
|
18674
|
+
source_uri: citation.source_uri ?? citation.artifact_uri ?? "unknown",
|
|
18675
|
+
quote: citation.quote,
|
|
18676
|
+
start_offset: citation.start_offset,
|
|
18677
|
+
end_offset: citation.end_offset,
|
|
18678
|
+
metadata: {
|
|
18679
|
+
source_ref: citation.source_ref,
|
|
18680
|
+
artifact_path: citation.artifact_path,
|
|
18681
|
+
revision: citation.revision,
|
|
18682
|
+
hash: citation.hash
|
|
18683
|
+
}
|
|
18684
|
+
})), now);
|
|
18685
|
+
upsertIndex(db, {
|
|
18686
|
+
title,
|
|
18687
|
+
path,
|
|
18688
|
+
artifactUri: artifact.uri,
|
|
18689
|
+
contentHash: artifact.hash ?? "",
|
|
18690
|
+
now
|
|
18691
|
+
});
|
|
18692
|
+
return {
|
|
18693
|
+
approved: true,
|
|
18694
|
+
durable_writes_performed: true,
|
|
18695
|
+
page_id: pageId,
|
|
18696
|
+
path,
|
|
18697
|
+
artifact_uri: artifact.uri,
|
|
18698
|
+
citations_written: written,
|
|
18699
|
+
log_key: log.key,
|
|
18700
|
+
message: `Filed answer to ${path}`
|
|
18701
|
+
};
|
|
18702
|
+
} finally {
|
|
18703
|
+
db.close();
|
|
18704
|
+
}
|
|
18705
|
+
}
|
|
18706
|
+
function addIssue(issues, issue2) {
|
|
18707
|
+
issues.push(issue2);
|
|
18708
|
+
}
|
|
18709
|
+
function lintWiki(options) {
|
|
18710
|
+
migrateKnowledgeDb(options.dbPath);
|
|
18711
|
+
const db = openKnowledgeDb(options.dbPath);
|
|
18712
|
+
const issues = [];
|
|
18713
|
+
try {
|
|
18714
|
+
const activePages = db.query("SELECT COUNT(*) AS n FROM wiki_pages WHERE status = 'active'").get()?.n ?? 0;
|
|
18715
|
+
const citationCount = db.query("SELECT COUNT(*) AS n FROM citations").get()?.n ?? 0;
|
|
18716
|
+
const backlinkCount = db.query("SELECT COUNT(*) AS n FROM wiki_backlinks").get()?.n ?? 0;
|
|
18717
|
+
const missingCitations = db.query(`SELECT wp.id, wp.path
|
|
18718
|
+
FROM wiki_pages wp
|
|
18719
|
+
LEFT JOIN citations c ON c.wiki_page_id = wp.id
|
|
18720
|
+
WHERE wp.status = 'active' AND wp.path LIKE 'wiki/generated/%'
|
|
18721
|
+
GROUP BY wp.id
|
|
18722
|
+
HAVING COUNT(c.id) = 0`).all();
|
|
18723
|
+
for (const page of missingCitations) {
|
|
18724
|
+
addIssue(issues, { type: "missing_citation", severity: "error", page_id: page.id, path: page.path, message: "Generated wiki page has no citations." });
|
|
18725
|
+
}
|
|
18726
|
+
const stale = db.query(`SELECT wp.id AS page_id, wp.path, c.source_uri, c.chunk_id
|
|
18727
|
+
FROM citations c
|
|
18728
|
+
JOIN wiki_pages wp ON wp.id = c.wiki_page_id
|
|
18729
|
+
LEFT JOIN chunks ch ON ch.id = c.chunk_id
|
|
18730
|
+
WHERE ch.metadata_json LIKE '%"stale":true%' OR ch.metadata_json LIKE '%"status":"stale"%' OR ch.metadata_json LIKE '%"status":"deleted"%'`).all();
|
|
18731
|
+
for (const row of stale) {
|
|
18732
|
+
addIssue(issues, { type: "stale_citation", severity: "warn", page_id: row.page_id, path: row.path, source_uri: row.source_uri, chunk_id: row.chunk_id ?? undefined, message: "Page cites a stale or deleted source chunk." });
|
|
18733
|
+
}
|
|
18734
|
+
const duplicates = db.query(`SELECT lower(title) AS title, COUNT(*) AS n
|
|
18735
|
+
FROM wiki_pages
|
|
18736
|
+
WHERE status = 'active'
|
|
18737
|
+
GROUP BY lower(title)
|
|
18738
|
+
HAVING COUNT(*) > 1`).all();
|
|
18739
|
+
for (const row of duplicates) {
|
|
18740
|
+
addIssue(issues, { type: "duplicate_page", severity: "warn", message: `Duplicate active wiki title: ${row.title} (${row.n} pages).` });
|
|
18741
|
+
}
|
|
18742
|
+
const orphans = db.query(`SELECT wp.id, wp.path
|
|
18743
|
+
FROM wiki_pages wp
|
|
18744
|
+
LEFT JOIN wiki_backlinks wb1 ON wb1.from_page_id = wp.id
|
|
18745
|
+
LEFT JOIN wiki_backlinks wb2 ON wb2.to_page_id = wp.id
|
|
18746
|
+
WHERE wp.status = 'active'
|
|
18747
|
+
AND wp.path NOT IN ('wiki/README.md')
|
|
18748
|
+
GROUP BY wp.id
|
|
18749
|
+
HAVING COUNT(wb1.to_page_id) = 0 AND COUNT(wb2.from_page_id) = 0`).all();
|
|
18750
|
+
for (const page of orphans) {
|
|
18751
|
+
addIssue(issues, { type: "orphan_page", severity: "info", page_id: page.id, path: page.path, message: "Wiki page has no backlinks." });
|
|
18752
|
+
}
|
|
18753
|
+
const unresolved = db.query(`SELECT wp.id AS page_id, wp.path, c.source_uri
|
|
18754
|
+
FROM citations c
|
|
18755
|
+
JOIN wiki_pages wp ON wp.id = c.wiki_page_id
|
|
18756
|
+
LEFT JOIN sources s ON s.uri = c.source_uri
|
|
18757
|
+
WHERE s.id IS NULL AND c.source_uri NOT LIKE 'file://%' AND c.source_uri NOT LIKE 's3://%' AND c.source_uri NOT LIKE 'https://%' AND c.source_uri NOT LIKE 'open-files://%'`).all();
|
|
18758
|
+
for (const row of unresolved) {
|
|
18759
|
+
addIssue(issues, { type: "unresolved_source_ref", severity: "error", page_id: row.page_id, path: row.path, source_uri: row.source_uri, message: "Citation source URI cannot be resolved to a known or allowed source ref." });
|
|
18760
|
+
}
|
|
18761
|
+
const contradictions = db.query(`SELECT id, path FROM wiki_pages WHERE lower(metadata_json) LIKE '%contradiction%'`).all();
|
|
18762
|
+
for (const page of contradictions) {
|
|
18763
|
+
addIssue(issues, { type: "contradiction_marker", severity: "warn", page_id: page.id, path: page.path, message: "Page metadata contains a contradiction marker." });
|
|
18764
|
+
}
|
|
18765
|
+
const newArticleCandidates = db.query(`SELECT c.id AS chunk_id, s.uri AS source_uri
|
|
18766
|
+
FROM chunks c
|
|
18767
|
+
JOIN source_revisions sr ON sr.id = c.source_revision_id
|
|
18768
|
+
JOIN sources s ON s.id = sr.source_id
|
|
18769
|
+
LEFT JOIN citations cit ON cit.chunk_id = c.id
|
|
18770
|
+
WHERE c.kind = 'source'
|
|
18771
|
+
GROUP BY c.id
|
|
18772
|
+
HAVING COUNT(cit.id) = 0
|
|
18773
|
+
LIMIT 25`).all();
|
|
18774
|
+
for (const row of newArticleCandidates) {
|
|
18775
|
+
addIssue(issues, { type: "new_article_candidate", severity: "info", chunk_id: row.chunk_id, source_uri: row.source_uri ?? undefined, message: "Source chunk is indexed but not cited by any wiki page yet." });
|
|
18776
|
+
}
|
|
18777
|
+
return {
|
|
18778
|
+
ok: issues.every((issue2) => issue2.severity !== "error"),
|
|
18779
|
+
issue_count: issues.length,
|
|
18780
|
+
issues,
|
|
18781
|
+
counts: {
|
|
18782
|
+
active_pages: activePages,
|
|
18783
|
+
citations: citationCount,
|
|
18784
|
+
backlinks: backlinkCount,
|
|
18785
|
+
new_article_candidates: newArticleCandidates.length
|
|
18786
|
+
}
|
|
18787
|
+
};
|
|
18788
|
+
} finally {
|
|
18789
|
+
db.close();
|
|
18790
|
+
}
|
|
18791
|
+
}
|
|
18792
|
+
|
|
18793
|
+
// src/wiki-layout.ts
|
|
18794
|
+
import { createHash as createHash11 } from "crypto";
|
|
18795
|
+
function todayParts2(now) {
|
|
17988
18796
|
const year = String(now.getUTCFullYear());
|
|
17989
18797
|
const month = String(now.getUTCMonth() + 1).padStart(2, "0");
|
|
17990
18798
|
const day = String(now.getUTCDate()).padStart(2, "0");
|
|
17991
18799
|
return { year, month, day };
|
|
17992
18800
|
}
|
|
17993
|
-
function
|
|
17994
|
-
return `${prefix}_${
|
|
18801
|
+
function stableId7(prefix, value) {
|
|
18802
|
+
return `${prefix}_${createHash11("sha256").update(value).digest("hex").slice(0, 20)}`;
|
|
17995
18803
|
}
|
|
17996
|
-
function
|
|
18804
|
+
function estimateTokenCount3(text) {
|
|
17997
18805
|
const words = text.trim().split(/\s+/).filter(Boolean).length;
|
|
17998
18806
|
return Math.max(1, Math.ceil(words * 1.25));
|
|
17999
18807
|
}
|
|
@@ -18052,7 +18860,7 @@ Pages should be concise, cited, and organized for both humans and agents.
|
|
|
18052
18860
|
`;
|
|
18053
18861
|
}
|
|
18054
18862
|
async function initializeWikiLayout(store, now = new Date) {
|
|
18055
|
-
const { year, month, day } =
|
|
18863
|
+
const { year, month, day } = todayParts2(now);
|
|
18056
18864
|
const schemaKey = "schemas/v1.md";
|
|
18057
18865
|
const rootIndexKey = "indexes/root.md";
|
|
18058
18866
|
const wikiReadmeKey = "wiki/README.md";
|
|
@@ -18109,7 +18917,7 @@ function provenanceFor(artifact) {
|
|
|
18109
18917
|
}
|
|
18110
18918
|
function recordWikiChunk(db, pageId, title, artifact, body, now) {
|
|
18111
18919
|
const provenance = provenanceFor(artifact);
|
|
18112
|
-
const chunkId =
|
|
18920
|
+
const chunkId = stableId7("chk", `${pageId}\x00${artifact.hash ?? artifact.uri}`);
|
|
18113
18921
|
const existing = db.query("SELECT id FROM chunks WHERE wiki_page_id = ?").all(pageId);
|
|
18114
18922
|
for (const row of existing)
|
|
18115
18923
|
db.run("DELETE FROM chunks_fts WHERE chunk_id = ?", [row.id]);
|
|
@@ -18121,7 +18929,7 @@ function recordWikiChunk(db, pageId, title, artifact, body, now) {
|
|
|
18121
18929
|
"wiki",
|
|
18122
18930
|
0,
|
|
18123
18931
|
body,
|
|
18124
|
-
|
|
18932
|
+
estimateTokenCount3(body),
|
|
18125
18933
|
0,
|
|
18126
18934
|
body.length,
|
|
18127
18935
|
JSON.stringify({
|
|
@@ -18145,7 +18953,7 @@ function recordWikiLayoutCatalog(db, artifacts, now = new Date) {
|
|
|
18145
18953
|
artifact_uri = excluded.artifact_uri,
|
|
18146
18954
|
metadata_json = excluded.metadata_json,
|
|
18147
18955
|
updated_at = excluded.updated_at`, [
|
|
18148
|
-
|
|
18956
|
+
stableId7("idx", "root:indexes/root.md"),
|
|
18149
18957
|
"root",
|
|
18150
18958
|
"root",
|
|
18151
18959
|
rootIndex.uri,
|
|
@@ -18160,7 +18968,7 @@ function recordWikiLayoutCatalog(db, artifacts, now = new Date) {
|
|
|
18160
18968
|
]);
|
|
18161
18969
|
}
|
|
18162
18970
|
if (wikiReadme) {
|
|
18163
|
-
const wikiPageId =
|
|
18971
|
+
const wikiPageId = stableId7("wiki", "wiki/README.md");
|
|
18164
18972
|
db.run(`INSERT INTO wiki_pages (id, path, title, artifact_uri, content_hash, status, metadata_json, created_at, updated_at)
|
|
18165
18973
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
18166
18974
|
ON CONFLICT(path) DO UPDATE SET
|
|
@@ -18188,6 +18996,17 @@ function recordWikiLayoutCatalog(db, artifacts, now = new Date) {
|
|
|
18188
18996
|
}
|
|
18189
18997
|
|
|
18190
18998
|
// src/service.ts
|
|
18999
|
+
function normalizeMode(value) {
|
|
19000
|
+
if (!value)
|
|
19001
|
+
return;
|
|
19002
|
+
const normalized = value.trim().toLowerCase();
|
|
19003
|
+
if (normalized === "local" || normalized === "offline")
|
|
19004
|
+
return "local";
|
|
19005
|
+
if (normalized === "hosted" || normalized === "remote" || normalized === "knowledge.hasna.xyz")
|
|
19006
|
+
return "hosted";
|
|
19007
|
+
throw new Error("Invalid setup mode. Use hosted or local.");
|
|
19008
|
+
}
|
|
19009
|
+
|
|
18191
19010
|
class KnowledgeService {
|
|
18192
19011
|
options;
|
|
18193
19012
|
ensuredWorkspace;
|
|
@@ -18228,6 +19047,59 @@ class KnowledgeService {
|
|
|
18228
19047
|
validateStorage() {
|
|
18229
19048
|
return validateStorageConfig(this.config(), this.ensureWorkspace());
|
|
18230
19049
|
}
|
|
19050
|
+
setup(options = {}) {
|
|
19051
|
+
const workspace = this.ensureWorkspace();
|
|
19052
|
+
const current = this.config();
|
|
19053
|
+
const mode = normalizeMode(options.mode) ?? current.mode;
|
|
19054
|
+
const apiUrl = options.apiUrl ? normalizeKnowledgeApiOrigin(options.apiUrl) : current.hosted?.api_url ? normalizeKnowledgeApiOrigin(current.hosted.api_url) : null;
|
|
19055
|
+
const nextConfig = {
|
|
19056
|
+
...current,
|
|
19057
|
+
mode,
|
|
19058
|
+
hosted: {
|
|
19059
|
+
...current.hosted ?? {},
|
|
19060
|
+
...apiUrl ? { api_url: apiUrl } : {}
|
|
19061
|
+
}
|
|
19062
|
+
};
|
|
19063
|
+
writeKnowledgeConfig(workspace.configPath, nextConfig);
|
|
19064
|
+
this.cachedConfig = nextConfig;
|
|
19065
|
+
return {
|
|
19066
|
+
ok: true,
|
|
19067
|
+
mode,
|
|
19068
|
+
api_url: nextConfig.hosted?.api_url ?? null,
|
|
19069
|
+
config_path: workspace.configPath,
|
|
19070
|
+
next: mode === "hosted" ? ["open-knowledge auth login --api-key <key>", "open-knowledge remote contracts --json"] : ["open-knowledge search <query>", "knowledge <prompt>"],
|
|
19071
|
+
message: `Set knowledge mode to ${mode}`
|
|
19072
|
+
};
|
|
19073
|
+
}
|
|
19074
|
+
authStatus(env = process.env) {
|
|
19075
|
+
return knowledgeAuthStatus(this.config(), env);
|
|
19076
|
+
}
|
|
19077
|
+
saveAuth(input, env = process.env) {
|
|
19078
|
+
const apiUrl = input.apiUrl ?? this.config().hosted?.api_url;
|
|
19079
|
+
return saveKnowledgeAuth({
|
|
19080
|
+
api_key: input.apiKey,
|
|
19081
|
+
email: input.email,
|
|
19082
|
+
org_id: input.orgId,
|
|
19083
|
+
org_slug: input.orgSlug,
|
|
19084
|
+
user_id: input.userId,
|
|
19085
|
+
api_url: apiUrl
|
|
19086
|
+
}, env);
|
|
19087
|
+
}
|
|
19088
|
+
clearAuth(env = process.env) {
|
|
19089
|
+
return clearKnowledgeAuth(env);
|
|
19090
|
+
}
|
|
19091
|
+
remoteContract() {
|
|
19092
|
+
const storage = this.storageContract();
|
|
19093
|
+
return knowledgeRegistryContract({
|
|
19094
|
+
mode: this.config().mode,
|
|
19095
|
+
sourceSchemes: this.config().sources.allowed_schemes,
|
|
19096
|
+
storageType: storage.artifact_store.type,
|
|
19097
|
+
artifactUriPrefix: storage.artifact_store.uri_prefix
|
|
19098
|
+
});
|
|
19099
|
+
}
|
|
19100
|
+
remoteClient(env = process.env) {
|
|
19101
|
+
return RemoteKnowledgeClient.fromConfig(this.config(), env);
|
|
19102
|
+
}
|
|
18231
19103
|
paths() {
|
|
18232
19104
|
const workspace = this.ensureWorkspace();
|
|
18233
19105
|
return {
|
|
@@ -18268,6 +19140,37 @@ class KnowledgeService {
|
|
|
18268
19140
|
}
|
|
18269
19141
|
return result;
|
|
18270
19142
|
}
|
|
19143
|
+
async compileWiki(options = {}) {
|
|
19144
|
+
const workspace = this.ensureWorkspace();
|
|
19145
|
+
return compileWikiPage({
|
|
19146
|
+
...options,
|
|
19147
|
+
dbPath: workspace.knowledgeDbPath,
|
|
19148
|
+
store: this.artifactStore()
|
|
19149
|
+
});
|
|
19150
|
+
}
|
|
19151
|
+
async fileAnswer(options) {
|
|
19152
|
+
const workspace = this.ensureWorkspace();
|
|
19153
|
+
const context = await this.retrieveContext({
|
|
19154
|
+
query: options.prompt,
|
|
19155
|
+
limit: options.limit,
|
|
19156
|
+
semantic: options.semantic,
|
|
19157
|
+
modelRef: options.modelRef,
|
|
19158
|
+
dimensions: options.dimensions,
|
|
19159
|
+
fake: options.fake
|
|
19160
|
+
});
|
|
19161
|
+
return fileAnswerToWiki({
|
|
19162
|
+
dbPath: workspace.knowledgeDbPath,
|
|
19163
|
+
store: this.artifactStore(),
|
|
19164
|
+
prompt: options.prompt,
|
|
19165
|
+
answer: options.answer,
|
|
19166
|
+
context,
|
|
19167
|
+
approveWrite: options.approveWrite
|
|
19168
|
+
});
|
|
19169
|
+
}
|
|
19170
|
+
lintWiki() {
|
|
19171
|
+
const workspace = this.ensureWorkspace();
|
|
19172
|
+
return lintWiki({ dbPath: workspace.knowledgeDbPath });
|
|
19173
|
+
}
|
|
18271
19174
|
async ingestManifest(input) {
|
|
18272
19175
|
const workspace = this.ensureWorkspace();
|
|
18273
19176
|
return ingestOpenFilesManifest({
|
|
@@ -18970,7 +19873,7 @@ function buildServer() {
|
|
|
18970
19873
|
const storePath = resolveStorePath(store_path, scope);
|
|
18971
19874
|
return readStoreLocked(storePath, (db) => {
|
|
18972
19875
|
const filePath = file2 || "./knowledge-export.json";
|
|
18973
|
-
|
|
19876
|
+
writeFileSync5(filePath, JSON.stringify(db, null, 2));
|
|
18974
19877
|
return jsonText({ ok: true, file: filePath, count: db.items.length });
|
|
18975
19878
|
});
|
|
18976
19879
|
});
|
|
@@ -18979,9 +19882,9 @@ function buildServer() {
|
|
|
18979
19882
|
store_path: storePathField,
|
|
18980
19883
|
scope: scopeField
|
|
18981
19884
|
}, async ({ file: file2, store_path, scope }) => {
|
|
18982
|
-
if (!
|
|
19885
|
+
if (!existsSync8(file2))
|
|
18983
19886
|
return errorText(`File not found: ${file2}`);
|
|
18984
|
-
const imported = JSON.parse(
|
|
19887
|
+
const imported = JSON.parse(readFileSync8(file2, "utf8"));
|
|
18985
19888
|
if (!imported || !Array.isArray(imported.items))
|
|
18986
19889
|
return errorText('Invalid import file: expected {"items": [...]}');
|
|
18987
19890
|
const storePath = resolveStorePath(store_path, scope);
|