@sourcepress/knowledge 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/.turbo/turbo-build.log +4 -0
  2. package/.turbo/turbo-test.log +21 -0
  3. package/dist/__tests__/graph-builder.test.d.ts +2 -0
  4. package/dist/__tests__/graph-builder.test.d.ts.map +1 -0
  5. package/dist/__tests__/graph-builder.test.js +122 -0
  6. package/dist/__tests__/graph-builder.test.js.map +1 -0
  7. package/dist/__tests__/graph-ops.test.d.ts +2 -0
  8. package/dist/__tests__/graph-ops.test.d.ts.map +1 -0
  9. package/dist/__tests__/graph-ops.test.js +181 -0
  10. package/dist/__tests__/graph-ops.test.js.map +1 -0
  11. package/dist/__tests__/ingestion.test.d.ts +2 -0
  12. package/dist/__tests__/ingestion.test.d.ts.map +1 -0
  13. package/dist/__tests__/ingestion.test.js +108 -0
  14. package/dist/__tests__/ingestion.test.js.map +1 -0
  15. package/dist/__tests__/json-file-store.test.d.ts +2 -0
  16. package/dist/__tests__/json-file-store.test.d.ts.map +1 -0
  17. package/dist/__tests__/json-file-store.test.js +180 -0
  18. package/dist/__tests__/json-file-store.test.js.map +1 -0
  19. package/dist/__tests__/knowledge-engine.test.d.ts +2 -0
  20. package/dist/__tests__/knowledge-engine.test.d.ts.map +1 -0
  21. package/dist/__tests__/knowledge-engine.test.js +152 -0
  22. package/dist/__tests__/knowledge-engine.test.js.map +1 -0
  23. package/dist/__tests__/knowledge-store.test.d.ts +2 -0
  24. package/dist/__tests__/knowledge-store.test.d.ts.map +1 -0
  25. package/dist/__tests__/knowledge-store.test.js +97 -0
  26. package/dist/__tests__/knowledge-store.test.js.map +1 -0
  27. package/dist/__tests__/scraper.test.d.ts +2 -0
  28. package/dist/__tests__/scraper.test.d.ts.map +1 -0
  29. package/dist/__tests__/scraper.test.js +66 -0
  30. package/dist/__tests__/scraper.test.js.map +1 -0
  31. package/dist/__tests__/sitemap-parser.test.d.ts +2 -0
  32. package/dist/__tests__/sitemap-parser.test.d.ts.map +1 -0
  33. package/dist/__tests__/sitemap-parser.test.js +75 -0
  34. package/dist/__tests__/sitemap-parser.test.js.map +1 -0
  35. package/dist/graph-builder.d.ts +17 -0
  36. package/dist/graph-builder.d.ts.map +1 -0
  37. package/dist/graph-builder.js +98 -0
  38. package/dist/graph-builder.js.map +1 -0
  39. package/dist/graph-ops.d.ts +21 -0
  40. package/dist/graph-ops.d.ts.map +1 -0
  41. package/dist/graph-ops.js +108 -0
  42. package/dist/graph-ops.js.map +1 -0
  43. package/dist/index.d.ts +10 -0
  44. package/dist/index.d.ts.map +1 -0
  45. package/dist/index.js +8 -0
  46. package/dist/index.js.map +1 -0
  47. package/dist/ingestion/index.d.ts +4 -0
  48. package/dist/ingestion/index.d.ts.map +1 -0
  49. package/dist/ingestion/index.js +3 -0
  50. package/dist/ingestion/index.js.map +1 -0
  51. package/dist/ingestion/scraper.d.ts +22 -0
  52. package/dist/ingestion/scraper.d.ts.map +1 -0
  53. package/dist/ingestion/scraper.js +118 -0
  54. package/dist/ingestion/scraper.js.map +1 -0
  55. package/dist/ingestion/sitemap-parser.d.ts +32 -0
  56. package/dist/ingestion/sitemap-parser.d.ts.map +1 -0
  57. package/dist/ingestion/sitemap-parser.js +104 -0
  58. package/dist/ingestion/sitemap-parser.js.map +1 -0
  59. package/dist/ingestion/types.d.ts +58 -0
  60. package/dist/ingestion/types.d.ts.map +1 -0
  61. package/dist/ingestion/types.js +2 -0
  62. package/dist/ingestion/types.js.map +1 -0
  63. package/dist/json-file-store.d.ts +19 -0
  64. package/dist/json-file-store.d.ts.map +1 -0
  65. package/dist/json-file-store.js +100 -0
  66. package/dist/json-file-store.js.map +1 -0
  67. package/dist/knowledge-engine.d.ts +45 -0
  68. package/dist/knowledge-engine.d.ts.map +1 -0
  69. package/dist/knowledge-engine.js +160 -0
  70. package/dist/knowledge-engine.js.map +1 -0
  71. package/dist/knowledge-store.d.ts +14 -0
  72. package/dist/knowledge-store.d.ts.map +1 -0
  73. package/dist/knowledge-store.js +40 -0
  74. package/dist/knowledge-store.js.map +1 -0
  75. package/dist/types.d.ts +67 -0
  76. package/dist/types.d.ts.map +1 -0
  77. package/dist/types.js +2 -0
  78. package/dist/types.js.map +1 -0
  79. package/package.json +26 -0
  80. package/src/__tests__/graph-builder.test.ts +129 -0
  81. package/src/__tests__/graph-ops.test.ts +189 -0
  82. package/src/__tests__/ingestion.test.ts +127 -0
  83. package/src/__tests__/json-file-store.test.ts +206 -0
  84. package/src/__tests__/knowledge-engine.test.ts +177 -0
  85. package/src/__tests__/knowledge-store.test.ts +111 -0
  86. package/src/__tests__/scraper.test.ts +74 -0
  87. package/src/__tests__/sitemap-parser.test.ts +85 -0
  88. package/src/graph-builder.ts +109 -0
  89. package/src/graph-ops.ts +129 -0
  90. package/src/index.ts +27 -0
  91. package/src/ingestion/index.ts +10 -0
  92. package/src/ingestion/scraper.ts +137 -0
  93. package/src/ingestion/sitemap-parser.ts +119 -0
  94. package/src/ingestion/types.ts +57 -0
  95. package/src/json-file-store.ts +127 -0
  96. package/src/knowledge-engine.ts +217 -0
  97. package/src/knowledge-store.ts +49 -0
  98. package/src/types.ts +76 -0
  99. package/tsconfig.json +5 -0
  100. package/vitest.config.ts +2 -0
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Injectable fetch function for testability.
3
+ * Matches the standard fetch signature (subset).
4
+ */
5
+ export type Fetcher = (url: string, init?: RequestInit) => Promise<Response>;
6
+ /**
7
+ * Result of scraping a single URL.
8
+ */
9
+ export interface ScrapeResult {
10
+ url: string;
11
+ title: string;
12
+ content: string;
13
+ markdown: string;
14
+ byline?: string;
15
+ excerpt?: string;
16
+ length: number;
17
+ scraped_at: string;
18
+ }
19
+ /**
20
+ * Sitemap section — a group of URLs sharing a path pattern.
21
+ */
22
+ export interface SitemapSection {
23
+ name: string;
24
+ pattern: string;
25
+ urls: string[];
26
+ count: number;
27
+ }
28
+ /**
29
+ * Parsed sitemap result — sections grouped by URL pattern.
30
+ */
31
+ export interface SitemapResult {
32
+ sitemap_url: string;
33
+ sections: SitemapSection[];
34
+ total_urls: number;
35
+ }
36
+ /**
37
+ * Options for running a sitemap import.
38
+ */
39
+ export interface SitemapRunOptions {
40
+ sitemap_url: string;
41
+ include?: string[];
42
+ exclude?: string[];
43
+ }
44
+ /**
45
+ * Progress info for batch operations.
46
+ */
47
+ export interface BatchProgress {
48
+ completed: number;
49
+ total: number;
50
+ failed: number;
51
+ results: Array<{
52
+ url: string;
53
+ path: string;
54
+ success: boolean;
55
+ error?: string;
56
+ }>;
57
+ }
58
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAChF"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/ingestion/types.ts"],"names":[],"mappings":""}
@@ -0,0 +1,19 @@
1
+ import type { KnowledgeFile } from "@sourcepress/core";
2
+ import type { KnowledgeFileFilter, KnowledgeGraph, KnowledgeStoreBackend } from "./types.js";
3
+ export declare class JsonFileStore implements KnowledgeStoreBackend {
4
+ private readonly knowledgePath;
5
+ private readonly graphPath;
6
+ private files;
7
+ private graph;
8
+ constructor(dir: string);
9
+ store(file: KnowledgeFile): Promise<void>;
10
+ retrieve(path: string): Promise<KnowledgeFile | null>;
11
+ list(filter?: KnowledgeFileFilter): Promise<KnowledgeFile[]>;
12
+ delete(path: string): Promise<boolean>;
13
+ count(): Promise<number>;
14
+ saveGraph(graph: KnowledgeGraph): Promise<void>;
15
+ loadGraph(): Promise<KnowledgeGraph | null>;
16
+ private loadFiles;
17
+ private writeFiles;
18
+ }
19
+ //# sourceMappingURL=json-file-store.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json-file-store.d.ts","sourceRoot":"","sources":["../src/json-file-store.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,KAAK,EAEX,mBAAmB,EACnB,cAAc,EACd,qBAAqB,EACrB,MAAM,YAAY,CAAC;AAUpB,qBAAa,aAAc,YAAW,qBAAqB;IAC1D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,OAAO,CAAC,KAAK,CAA2C;IACxD,OAAO,CAAC,KAAK,CAAgD;gBAEjD,GAAG,EAAE,MAAM;IASjB,KAAK,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAIrD,IAAI,CAAC,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkB5D,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAStC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAY/C,SAAS,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAoBjD,OAAO,CAAC,SAAS;IAcjB,OAAO,CAAC,UAAU;CAKlB"}
@@ -0,0 +1,100 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ export class JsonFileStore {
4
+ knowledgePath;
5
+ graphPath;
6
+ files = null;
7
+ graph = undefined;
8
+ constructor(dir) {
9
+ this.knowledgePath = join(dir, "knowledge.json");
10
+ this.graphPath = join(dir, "graph.json");
11
+ if (!existsSync(dir)) {
12
+ mkdirSync(dir, { recursive: true });
13
+ }
14
+ }
15
+ async store(file) {
16
+ const files = this.loadFiles();
17
+ files.set(file.path, { ...file });
18
+ this.writeFiles(files);
19
+ }
20
+ async retrieve(path) {
21
+ return this.loadFiles().get(path) ?? null;
22
+ }
23
+ async list(filter) {
24
+ let results = Array.from(this.loadFiles().values());
25
+ if (filter?.type) {
26
+ results = results.filter((f) => f.type === filter.type);
27
+ }
28
+ if (filter?.quality) {
29
+ results = results.filter((f) => f.quality === filter.quality);
30
+ }
31
+ if (filter?.source) {
32
+ results = results.filter((f) => f.source === filter.source);
33
+ }
34
+ if (filter?.since) {
35
+ const since = filter.since;
36
+ results = results.filter((f) => f.ingested_at >= since);
37
+ }
38
+ return results;
39
+ }
40
+ async delete(path) {
41
+ const files = this.loadFiles();
42
+ const deleted = files.delete(path);
43
+ if (deleted) {
44
+ this.writeFiles(files);
45
+ }
46
+ return deleted;
47
+ }
48
+ async count() {
49
+ return this.loadFiles().size;
50
+ }
51
+ async saveGraph(graph) {
52
+ this.graph = graph;
53
+ const serialized = {
54
+ entities: Array.from(graph.entities.entries()),
55
+ relations: graph.relations,
56
+ clusters: graph.clusters,
57
+ built_at: graph.built_at,
58
+ file_count: graph.file_count,
59
+ };
60
+ writeFileSync(this.graphPath, JSON.stringify(serialized, null, "\t"), "utf-8");
61
+ }
62
+ async loadGraph() {
63
+ if (this.graph !== undefined) {
64
+ return this.graph;
65
+ }
66
+ if (!existsSync(this.graphPath)) {
67
+ this.graph = null;
68
+ return null;
69
+ }
70
+ const raw = readFileSync(this.graphPath, "utf-8");
71
+ const data = JSON.parse(raw);
72
+ this.graph = {
73
+ entities: new Map(data.entities),
74
+ relations: data.relations,
75
+ clusters: data.clusters,
76
+ built_at: data.built_at,
77
+ file_count: data.file_count,
78
+ };
79
+ return this.graph;
80
+ }
81
+ loadFiles() {
82
+ if (this.files !== null) {
83
+ return this.files;
84
+ }
85
+ if (!existsSync(this.knowledgePath)) {
86
+ this.files = new Map();
87
+ return this.files;
88
+ }
89
+ const raw = readFileSync(this.knowledgePath, "utf-8");
90
+ const entries = JSON.parse(raw);
91
+ this.files = new Map(entries);
92
+ return this.files;
93
+ }
94
+ writeFiles(files) {
95
+ this.files = files;
96
+ const entries = Array.from(files.entries());
97
+ writeFileSync(this.knowledgePath, JSON.stringify(entries, null, "\t"), "utf-8");
98
+ }
99
+ }
100
+ //# sourceMappingURL=json-file-store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"json-file-store.js","sourceRoot":"","sources":["../src/json-file-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiBjC,MAAM,OAAO,aAAa;IACR,aAAa,CAAS;IACtB,SAAS,CAAS;IAE3B,KAAK,GAAsC,IAAI,CAAC;IAChD,KAAK,GAAsC,SAAS,CAAC;IAE7D,YAAY,GAAW;QACtB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAmB;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC/B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY;QAC1B,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAA4B;QACtC,IAAI,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QACpD,IAAI,MAAM,EAAE,IAAI,EAAE,CAAC;YAClB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACrB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;YACpB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,MAAM,EAAE,KAAK,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAY;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,IAAI,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAqB;QACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,MAAM,UAAU,GAAoB;YACnC,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YAC9C,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,UAAU,EAAE,KAAK,CAAC,UAAU;SAC5B,CAAC;QACF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,SAAS;QACd,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,KAAK,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAoB,CAAC;QAChD,IAAI,CAAC,KAAK,GAAG;YACZ,QAAQ,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;YAChC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC3B,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;IAEO,SAAS;QAChB,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,KAAK,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,KAAK,CAAC;QACnB,CAAC;QACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAmC,CAAC;QAClE,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QAC9B,OAAO,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;IAEO,UAAU,CAAC,KAAiC;QACnD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5C,aAAa,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;IACjF,CAAC;CACD"}
@@ -0,0 +1,45 @@
1
+ import type { BudgetTracker, ResolvedProvider } from "@sourcepress/ai";
2
+ import type { ContentFile, KnowledgeFile } from "@sourcepress/core";
3
+ import type { BatchProgress, Fetcher, SitemapResult, SitemapRunOptions } from "./ingestion/types.js";
4
+ import type { GraphQueryResult, KnowledgeGap, KnowledgeGraph, KnowledgeStoreBackend, StaleContent } from "./types.js";
5
+ export declare class KnowledgeEngine {
6
+ private store;
7
+ private provider;
8
+ private budget;
9
+ private graphOps;
10
+ private currentGraph;
11
+ private scraper;
12
+ private sitemapParser;
13
+ constructor(store: KnowledgeStoreBackend, provider: ResolvedProvider, budget: BudgetTracker, fetcher?: Fetcher);
14
+ initialize(): Promise<void>;
15
+ ingest(path: string, body: string, source: KnowledgeFile["source"], sourceUrl?: string): Promise<KnowledgeFile>;
16
+ buildGraph(): Promise<KnowledgeGraph>;
17
+ query(nameOrAlias: string): GraphQueryResult | null;
18
+ findGaps(contentFiles: ContentFile[]): KnowledgeGap[];
19
+ findStale(contentFiles: ContentFile[], knowledgeTimestamps: Record<string, string>): StaleContent[];
20
+ getGraph(): KnowledgeGraph | null;
21
+ getStore(): KnowledgeStoreBackend;
22
+ /**
23
+ * Import a single URL: scrape → classify → extract entities → store.
24
+ */
25
+ importUrl(url: string): Promise<KnowledgeFile>;
26
+ /**
27
+ * Parse a sitemap and return grouped sections for interactive selection.
28
+ */
29
+ parseSitemap(sitemapUrl: string): Promise<SitemapResult>;
30
+ /**
31
+ * Filter a parsed sitemap result by include/exclude patterns.
32
+ */
33
+ filterSitemapUrls(result: SitemapResult, options: SitemapRunOptions): string[];
34
+ /**
35
+ * Import a batch of URLs. Calls progressCallback for each completed URL.
36
+ */
37
+ importBatch(urls: string[], progressCallback?: (completed: number, total: number, failed: number) => void): Promise<BatchProgress>;
38
+ /**
39
+ * Convert a URL to a knowledge file path.
40
+ * https://example.com/blog/post-1 → knowledge/example.com/blog/post-1.md
41
+ */
42
+ private urlToPath;
43
+ private getExistingEntities;
44
+ }
45
+ //# sourceMappingURL=knowledge-engine.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"knowledge-engine.d.ts","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAKpE,OAAO,KAAK,EACX,aAAa,EACb,OAAO,EACP,aAAa,EACb,iBAAiB,EACjB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAEX,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,qBAAqB,EACrB,YAAY,EACZ,MAAM,YAAY,CAAC;AAEpB,qBAAa,eAAe;IAC3B,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,QAAQ,CAAmB;IACnC,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,QAAQ,CAAyB;IACzC,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,aAAa,CAAgB;gBAGpC,KAAK,EAAE,qBAAqB,EAC5B,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,aAAa,EACrB,OAAO,CAAC,EAAE,OAAO;IASZ,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,MAAM,CACX,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,EAC/B,SAAS,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,aAAa,CAAC;IAwBnB,UAAU,IAAI,OAAO,CAAC,cAAc,CAAC;IAkC3C,KAAK,CAAC,WAAW,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;IAKnD,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE,GAAG,YAAY,EAAE;IAKrD,SAAS,CACR,YAAY,EAAE,WAAW,EAAE,EAC3B,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACzC,YAAY,EAAE;IAKjB,QAAQ,IAAI,cAAc,GAAG,IAAI;IAGjC,QAAQ,IAAI,qBAAqB;IAIjC;;OAEG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAMpD;;OAEG;IACG,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAI9D;;OAEG;IACH,iBAAiB,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,iBAAiB,GAAG,MAAM,EAAE;IAI9E;;OAEG;IACG,WAAW,CAChB,IAAI,EAAE,MAAM,EAAE,EACd,gBAAgB,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,GAC3E,OAAO,CAAC,aAAa,CAAC;IA6BzB;;;OAGG;IACH,OAAO,CAAC,SAAS;IAOjB,OAAO,CAAC,mBAAmB;CAO3B"}
@@ -0,0 +1,160 @@
1
+ import { classify, extract } from "@sourcepress/ai";
2
+ import { GraphBuilder } from "./graph-builder.js";
3
+ import { GraphOps } from "./graph-ops.js";
4
+ import { Scraper } from "./ingestion/scraper.js";
5
+ import { SitemapParser } from "./ingestion/sitemap-parser.js";
6
+ export class KnowledgeEngine {
7
+ store;
8
+ provider;
9
+ budget;
10
+ graphOps = null;
11
+ currentGraph = null;
12
+ scraper;
13
+ sitemapParser;
14
+ constructor(store, provider, budget, fetcher) {
15
+ this.store = store;
16
+ this.provider = provider;
17
+ this.budget = budget;
18
+ this.scraper = new Scraper(fetcher);
19
+ this.sitemapParser = new SitemapParser(fetcher);
20
+ }
21
+ async initialize() {
22
+ const graph = await this.store.loadGraph();
23
+ if (graph) {
24
+ this.currentGraph = graph;
25
+ this.graphOps = new GraphOps(graph);
26
+ }
27
+ }
28
+ async ingest(path, body, source, sourceUrl) {
29
+ const classification = await classify({ text: body }, this.provider, this.budget);
30
+ const extraction = await extract({ text: body, file_path: path, existing_entities: this.getExistingEntities() }, this.provider, this.budget);
31
+ const knowledgeFile = {
32
+ path,
33
+ type: classification.type,
34
+ quality: classification.quality,
35
+ quality_score: classification.quality_score,
36
+ entities: extraction.entities.map((e) => ({ type: e.type, name: e.name })),
37
+ ingested_at: new Date().toISOString(),
38
+ source,
39
+ source_url: sourceUrl,
40
+ body,
41
+ };
42
+ await this.store.store(knowledgeFile);
43
+ return knowledgeFile;
44
+ }
45
+ async buildGraph() {
46
+ const files = await this.store.list();
47
+ const builder = new GraphBuilder();
48
+ for (const file of files) {
49
+ const extraction = await extract({
50
+ text: file.body,
51
+ file_path: file.path,
52
+ existing_entities: Array.from(builder.getEntities().values()),
53
+ }, this.provider, this.budget);
54
+ const entities = extraction.entities.map((e) => ({
55
+ type: e.type,
56
+ name: e.name,
57
+ aliases: e.aliases ?? [],
58
+ confidence: e.confidence ?? 0.5,
59
+ source_file: file.path,
60
+ }));
61
+ builder.addEntities(entities);
62
+ builder.addRelations(extraction.relations.map((r) => ({ ...r, source_file: file.path })));
63
+ }
64
+ const graph = builder.build();
65
+ this.currentGraph = graph;
66
+ this.graphOps = new GraphOps(graph);
67
+ await this.store.saveGraph(graph);
68
+ return graph;
69
+ }
70
+ query(nameOrAlias) {
71
+ if (!this.graphOps)
72
+ return null;
73
+ return this.graphOps.query(nameOrAlias);
74
+ }
75
+ findGaps(contentFiles) {
76
+ if (!this.graphOps)
77
+ return [];
78
+ return this.graphOps.findGaps(contentFiles);
79
+ }
80
+ findStale(contentFiles, knowledgeTimestamps) {
81
+ if (!this.graphOps)
82
+ return [];
83
+ return this.graphOps.findStale(contentFiles, knowledgeTimestamps);
84
+ }
85
+ getGraph() {
86
+ return this.currentGraph;
87
+ }
88
+ getStore() {
89
+ return this.store;
90
+ }
91
+ /**
92
+ * Import a single URL: scrape → classify → extract entities → store.
93
+ */
94
+ async importUrl(url) {
95
+ const scraped = await this.scraper.scrape(url);
96
+ const path = this.urlToPath(url);
97
+ return this.ingest(path, scraped.markdown, "url", url);
98
+ }
99
+ /**
100
+ * Parse a sitemap and return grouped sections for interactive selection.
101
+ */
102
+ async parseSitemap(sitemapUrl) {
103
+ return this.sitemapParser.parse(sitemapUrl);
104
+ }
105
+ /**
106
+ * Filter a parsed sitemap result by include/exclude patterns.
107
+ */
108
+ filterSitemapUrls(result, options) {
109
+ return this.sitemapParser.filterUrls(result, options);
110
+ }
111
+ /**
112
+ * Import a batch of URLs. Calls progressCallback for each completed URL.
113
+ */
114
+ async importBatch(urls, progressCallback) {
115
+ const progress = {
116
+ completed: 0,
117
+ total: urls.length,
118
+ failed: 0,
119
+ results: [],
120
+ };
121
+ for (const url of urls) {
122
+ try {
123
+ const file = await this.importUrl(url);
124
+ progress.completed++;
125
+ progress.results.push({ url, path: file.path, success: true });
126
+ }
127
+ catch (error) {
128
+ progress.failed++;
129
+ progress.completed++;
130
+ progress.results.push({
131
+ url,
132
+ path: "",
133
+ success: false,
134
+ error: error instanceof Error ? error.message : String(error),
135
+ });
136
+ }
137
+ progressCallback?.(progress.completed, progress.total, progress.failed);
138
+ }
139
+ return progress;
140
+ }
141
+ /**
142
+ * Convert a URL to a knowledge file path.
143
+ * https://example.com/blog/post-1 → knowledge/example.com/blog/post-1.md
144
+ */
145
+ urlToPath(url) {
146
+ const parsed = new URL(url);
147
+ const pathname = parsed.pathname.replace(/\/$/, "") || "/index";
148
+ const clean = pathname.replace(/\.[^.]+$/, ""); // Remove file extensions
149
+ return `knowledge/${parsed.hostname}${clean}.md`;
150
+ }
151
+ getExistingEntities() {
152
+ if (!this.currentGraph)
153
+ return [];
154
+ return Array.from(this.currentGraph.entities.values()).map((e) => ({
155
+ type: e.type,
156
+ name: e.name,
157
+ }));
158
+ }
159
+ }
160
+ //# sourceMappingURL=knowledge-engine.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"knowledge-engine.js","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAEpD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAgB9D,MAAM,OAAO,eAAe;IACnB,KAAK,CAAwB;IAC7B,QAAQ,CAAmB;IAC3B,MAAM,CAAgB;IACtB,QAAQ,GAAoB,IAAI,CAAC;IACjC,YAAY,GAA0B,IAAI,CAAC;IAC3C,OAAO,CAAU;IACjB,aAAa,CAAgB;IAErC,YACC,KAA4B,EAC5B,QAA0B,EAC1B,MAAqB,EACrB,OAAiB;QAEjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,UAAU;QACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;YAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CACX,IAAY,EACZ,IAAY,EACZ,MAA+B,EAC/B,SAAkB;QAElB,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,MAAM,OAAO,CAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,CAAC,mBAAmB,EAAE,EAAE,EAC9E,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,MAAM,CACX,CAAC;QAEF,MAAM,aAAa,GAAkB;YACpC,IAAI;YACJ,IAAI,EAAE,cAAc,CAAC,IAAI;YACzB,OAAO,EAAE,cAAc,CAAC,OAAO;YAC/B,aAAa,EAAE,cAAc,CAAC,aAAa;YAC3C,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1E,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM;YACN,UAAU,EAAE,SAAS;YACrB,IAAI;SACJ,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACtC,OAAO,aAAa,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,UAAU;QACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;QAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,UAAU,GAAG,MAAM,OAAO,CAC/B;gBACC,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,SAAS,EAAE,IAAI,CAAC,IAAI;gBACpB,iBAAiB,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,CAAC;aAC7D,EACD,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,MAAM,CACX,CAAC;YAEF,MAAM,QAAQ,GAAsB,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACnE,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;gBACxB,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,GAAG;gBAC/B,WAAW,EAAE,IAAI,CAAC,IAAI;aACtB,CAAC,CAAC,CAAC;YAEJ,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC9B,OAAO,CAAC,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3F,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IAED,KAAK,CAAC,WAAmB;QACxB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzC,CAAC;IAED,QAAQ,CAAC,YAA2B;QACnC,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC7C,CAAC;IAED,SAAS,CACR,YAA2B,EAC3B,mBAA2C;QAE3C,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;IACnE,CAAC;IAED,QAAQ;QACP,OAAO,IAAI,CAAC,YAAY,CAAC;IAC1B,CAAC;IACD,QAAQ;QACP,OAAO,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW;QAC1B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,UAAkB;QACpC,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,MAAqB,EAAE,OAA0B;QAClE,OAAO,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAChB,IAAc,EACd,gBAA6E;QAE7E,MAAM,QAAQ,GAAkB;YAC/B,SAAS,EAAE,CAAC;YACZ,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,MAAM,EAAE,CAAC;YACT,OAAO,EAAE,EAAE;SACX,CAAC;QAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC;gBACJ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACvC,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACrB,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,QAAQ,CAAC,SAAS,EAAE,CAAC;gBACrB,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;oBACrB,GAAG;oBACH,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC7D,CAAC,CAAC;YACJ,CAAC;YACD,gBAAgB,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED;;;OAGG;IACK,SAAS,CAAC,GAAW;QAC5B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,QAAQ,CAAC;QAChE,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,yBAAyB;QACzE,OAAO,aAAa,MAAM,CAAC,QAAQ,GAAG,KAAK,KAAK,CAAC;IAClD,CAAC;IAEO,mBAAmB;QAC1B,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClE,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,IAAI,EAAE,CAAC,CAAC,IAAI;SACZ,CAAC,CAAC,CAAC;IACL,CAAC;CACD"}
@@ -0,0 +1,14 @@
1
+ import type { KnowledgeFile } from "@sourcepress/core";
2
+ import type { KnowledgeFileFilter, KnowledgeGraph, KnowledgeStoreBackend } from "./types.js";
3
+ export declare class InMemoryKnowledgeStore implements KnowledgeStoreBackend {
4
+ private files;
5
+ private graph;
6
+ store(file: KnowledgeFile): Promise<void>;
7
+ retrieve(path: string): Promise<KnowledgeFile | null>;
8
+ list(filter?: KnowledgeFileFilter): Promise<KnowledgeFile[]>;
9
+ delete(path: string): Promise<boolean>;
10
+ count(): Promise<number>;
11
+ saveGraph(graph: KnowledgeGraph): Promise<void>;
12
+ loadGraph(): Promise<KnowledgeGraph | null>;
13
+ }
14
+ //# sourceMappingURL=knowledge-store.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"knowledge-store.d.ts","sourceRoot":"","sources":["../src/knowledge-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,KAAK,EAAE,mBAAmB,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAE7F,qBAAa,sBAAuB,YAAW,qBAAqB;IACnE,OAAO,CAAC,KAAK,CAAyC;IACtD,OAAO,CAAC,KAAK,CAA+B;IAEtC,KAAK,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAIrD,IAAI,CAAC,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkB5D,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAItC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/C,SAAS,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;CAGjD"}
@@ -0,0 +1,40 @@
1
+ export class InMemoryKnowledgeStore {
2
+ files = new Map();
3
+ graph = null;
4
+ async store(file) {
5
+ this.files.set(file.path, { ...file });
6
+ }
7
+ async retrieve(path) {
8
+ return this.files.get(path) ?? null;
9
+ }
10
+ async list(filter) {
11
+ let results = Array.from(this.files.values());
12
+ if (filter?.type) {
13
+ results = results.filter((f) => f.type === filter.type);
14
+ }
15
+ if (filter?.quality) {
16
+ results = results.filter((f) => f.quality === filter.quality);
17
+ }
18
+ if (filter?.source) {
19
+ results = results.filter((f) => f.source === filter.source);
20
+ }
21
+ if (filter?.since) {
22
+ const since = filter.since;
23
+ results = results.filter((f) => f.ingested_at >= since);
24
+ }
25
+ return results;
26
+ }
27
+ async delete(path) {
28
+ return this.files.delete(path);
29
+ }
30
+ async count() {
31
+ return this.files.size;
32
+ }
33
+ async saveGraph(graph) {
34
+ this.graph = graph;
35
+ }
36
+ async loadGraph() {
37
+ return this.graph;
38
+ }
39
+ }
40
+ //# sourceMappingURL=knowledge-store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"knowledge-store.js","sourceRoot":"","sources":["../src/knowledge-store.ts"],"names":[],"mappings":"AAGA,MAAM,OAAO,sBAAsB;IAC1B,KAAK,GAA+B,IAAI,GAAG,EAAE,CAAC;IAC9C,KAAK,GAA0B,IAAI,CAAC;IAE5C,KAAK,CAAC,KAAK,CAAC,IAAmB;QAC9B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAA4B;QACtC,IAAI,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,IAAI,MAAM,EAAE,IAAI,EAAE,CAAC;YAClB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACrB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;YACpB,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,MAAM,EAAE,KAAK,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YAC3B,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,KAAK,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAqB;QACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,SAAS;QACd,OAAO,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;CACD"}
@@ -0,0 +1,67 @@
1
+ import type { Entity, KnowledgeFile } from "@sourcepress/core";
2
+ export interface ExtractionResult {
3
+ entities: ExtractedEntity[];
4
+ relations: ExtractedRelation[];
5
+ }
6
+ export interface ExtractedEntity extends Entity {
7
+ aliases: string[];
8
+ confidence: number;
9
+ source_file: string;
10
+ }
11
+ export interface ExtractedRelation {
12
+ from_entity: string;
13
+ to_entity: string;
14
+ relation_type: string;
15
+ confidence: number;
16
+ evidence: string;
17
+ source_file: string;
18
+ }
19
+ export interface EntityCluster {
20
+ id: string;
21
+ name: string;
22
+ entities: string[];
23
+ coherence_score: number;
24
+ }
25
+ export interface KnowledgeGraph {
26
+ entities: Map<string, ExtractedEntity>;
27
+ relations: ExtractedRelation[];
28
+ clusters: EntityCluster[];
29
+ built_at: string;
30
+ file_count: number;
31
+ }
32
+ export interface GraphQueryResult {
33
+ entity: ExtractedEntity;
34
+ relations: ExtractedRelation[];
35
+ related_entities: ExtractedEntity[];
36
+ files: string[];
37
+ }
38
+ export interface KnowledgeGap {
39
+ entity_name: string;
40
+ entity_type: string;
41
+ knowledge_file_count: number;
42
+ content_file_count: number;
43
+ reason: string;
44
+ }
45
+ export interface StaleContent {
46
+ content_path: string;
47
+ generated_at: string;
48
+ newest_source_change: string;
49
+ stale_sources: string[];
50
+ reason: string;
51
+ }
52
+ export interface KnowledgeStoreBackend {
53
+ store(file: KnowledgeFile): Promise<void>;
54
+ retrieve(path: string): Promise<KnowledgeFile | null>;
55
+ list(filter?: KnowledgeFileFilter): Promise<KnowledgeFile[]>;
56
+ delete(path: string): Promise<boolean>;
57
+ count(): Promise<number>;
58
+ saveGraph(graph: KnowledgeGraph): Promise<void>;
59
+ loadGraph(): Promise<KnowledgeGraph | null>;
60
+ }
61
+ export interface KnowledgeFileFilter {
62
+ type?: string;
63
+ quality?: KnowledgeFile["quality"];
64
+ source?: KnowledgeFile["source"];
65
+ since?: string;
66
+ }
67
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAa,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAE1E,MAAM,WAAW,gBAAgB;IAChC,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,SAAS,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,eAAgB,SAAQ,MAAM;IAC9C,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACvC,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,KAAK,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,qBAAqB;IACrC,KAAK,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,MAAM,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IAC7D,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IACzB,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,SAAS,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;CAC5C;AAED,MAAM,WAAW,mBAAmB;IACnC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,CAAC,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CACf"}
package/dist/types.js ADDED
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
package/package.json ADDED
@@ -0,0 +1,26 @@
1
+ {
2
+ "name": "@sourcepress/knowledge",
3
+ "version": "0.1.0",
4
+ "publishConfig": { "access": "public" },
5
+ "type": "module",
6
+ "exports": {
7
+ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" }
8
+ },
9
+ "scripts": {
10
+ "build": "tsc",
11
+ "test": "vitest run",
12
+ "typecheck": "tsc --noEmit",
13
+ "clean": "rm -rf dist"
14
+ },
15
+ "dependencies": {
16
+ "@sourcepress/core": "workspace:*",
17
+ "@sourcepress/ai": "workspace:*",
18
+ "linkedom": "^0.18.0",
19
+ "@mozilla/readability": "^0.5.0"
20
+ },
21
+ "devDependencies": {
22
+ "typescript": "^5.7.0",
23
+ "vitest": "^3.0.0",
24
+ "@types/mozilla__readability": "^0.4.0"
25
+ }
26
+ }