@sourcepress/knowledge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -0
- package/.turbo/turbo-test.log +21 -0
- package/dist/__tests__/graph-builder.test.d.ts +2 -0
- package/dist/__tests__/graph-builder.test.d.ts.map +1 -0
- package/dist/__tests__/graph-builder.test.js +122 -0
- package/dist/__tests__/graph-builder.test.js.map +1 -0
- package/dist/__tests__/graph-ops.test.d.ts +2 -0
- package/dist/__tests__/graph-ops.test.d.ts.map +1 -0
- package/dist/__tests__/graph-ops.test.js +181 -0
- package/dist/__tests__/graph-ops.test.js.map +1 -0
- package/dist/__tests__/ingestion.test.d.ts +2 -0
- package/dist/__tests__/ingestion.test.d.ts.map +1 -0
- package/dist/__tests__/ingestion.test.js +108 -0
- package/dist/__tests__/ingestion.test.js.map +1 -0
- package/dist/__tests__/json-file-store.test.d.ts +2 -0
- package/dist/__tests__/json-file-store.test.d.ts.map +1 -0
- package/dist/__tests__/json-file-store.test.js +180 -0
- package/dist/__tests__/json-file-store.test.js.map +1 -0
- package/dist/__tests__/knowledge-engine.test.d.ts +2 -0
- package/dist/__tests__/knowledge-engine.test.d.ts.map +1 -0
- package/dist/__tests__/knowledge-engine.test.js +152 -0
- package/dist/__tests__/knowledge-engine.test.js.map +1 -0
- package/dist/__tests__/knowledge-store.test.d.ts +2 -0
- package/dist/__tests__/knowledge-store.test.d.ts.map +1 -0
- package/dist/__tests__/knowledge-store.test.js +97 -0
- package/dist/__tests__/knowledge-store.test.js.map +1 -0
- package/dist/__tests__/scraper.test.d.ts +2 -0
- package/dist/__tests__/scraper.test.d.ts.map +1 -0
- package/dist/__tests__/scraper.test.js +66 -0
- package/dist/__tests__/scraper.test.js.map +1 -0
- package/dist/__tests__/sitemap-parser.test.d.ts +2 -0
- package/dist/__tests__/sitemap-parser.test.d.ts.map +1 -0
- package/dist/__tests__/sitemap-parser.test.js +75 -0
- package/dist/__tests__/sitemap-parser.test.js.map +1 -0
- package/dist/graph-builder.d.ts +17 -0
- package/dist/graph-builder.d.ts.map +1 -0
- package/dist/graph-builder.js +98 -0
- package/dist/graph-builder.js.map +1 -0
- package/dist/graph-ops.d.ts +21 -0
- package/dist/graph-ops.d.ts.map +1 -0
- package/dist/graph-ops.js +108 -0
- package/dist/graph-ops.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/ingestion/index.d.ts +4 -0
- package/dist/ingestion/index.d.ts.map +1 -0
- package/dist/ingestion/index.js +3 -0
- package/dist/ingestion/index.js.map +1 -0
- package/dist/ingestion/scraper.d.ts +22 -0
- package/dist/ingestion/scraper.d.ts.map +1 -0
- package/dist/ingestion/scraper.js +118 -0
- package/dist/ingestion/scraper.js.map +1 -0
- package/dist/ingestion/sitemap-parser.d.ts +32 -0
- package/dist/ingestion/sitemap-parser.d.ts.map +1 -0
- package/dist/ingestion/sitemap-parser.js +104 -0
- package/dist/ingestion/sitemap-parser.js.map +1 -0
- package/dist/ingestion/types.d.ts +58 -0
- package/dist/ingestion/types.d.ts.map +1 -0
- package/dist/ingestion/types.js +2 -0
- package/dist/ingestion/types.js.map +1 -0
- package/dist/json-file-store.d.ts +19 -0
- package/dist/json-file-store.d.ts.map +1 -0
- package/dist/json-file-store.js +100 -0
- package/dist/json-file-store.js.map +1 -0
- package/dist/knowledge-engine.d.ts +45 -0
- package/dist/knowledge-engine.d.ts.map +1 -0
- package/dist/knowledge-engine.js +160 -0
- package/dist/knowledge-engine.js.map +1 -0
- package/dist/knowledge-store.d.ts +14 -0
- package/dist/knowledge-store.d.ts.map +1 -0
- package/dist/knowledge-store.js +40 -0
- package/dist/knowledge-store.js.map +1 -0
- package/dist/types.d.ts +67 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +26 -0
- package/src/__tests__/graph-builder.test.ts +129 -0
- package/src/__tests__/graph-ops.test.ts +189 -0
- package/src/__tests__/ingestion.test.ts +127 -0
- package/src/__tests__/json-file-store.test.ts +206 -0
- package/src/__tests__/knowledge-engine.test.ts +177 -0
- package/src/__tests__/knowledge-store.test.ts +111 -0
- package/src/__tests__/scraper.test.ts +74 -0
- package/src/__tests__/sitemap-parser.test.ts +85 -0
- package/src/graph-builder.ts +109 -0
- package/src/graph-ops.ts +129 -0
- package/src/index.ts +27 -0
- package/src/ingestion/index.ts +10 -0
- package/src/ingestion/scraper.ts +137 -0
- package/src/ingestion/sitemap-parser.ts +119 -0
- package/src/ingestion/types.ts +57 -0
- package/src/json-file-store.ts +127 -0
- package/src/knowledge-engine.ts +217 -0
- package/src/knowledge-store.ts +49 -0
- package/src/types.ts +76 -0
- package/tsconfig.json +5 -0
- package/vitest.config.ts +2 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { KnowledgeFile } from "@sourcepress/core";
|
|
2
|
+
import type { KnowledgeFileFilter, KnowledgeGraph, KnowledgeStoreBackend } from "./types.js";
|
|
3
|
+
|
|
4
|
+
export class InMemoryKnowledgeStore implements KnowledgeStoreBackend {
|
|
5
|
+
private files: Map<string, KnowledgeFile> = new Map();
|
|
6
|
+
private graph: KnowledgeGraph | null = null;
|
|
7
|
+
|
|
8
|
+
async store(file: KnowledgeFile): Promise<void> {
|
|
9
|
+
this.files.set(file.path, { ...file });
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async retrieve(path: string): Promise<KnowledgeFile | null> {
|
|
13
|
+
return this.files.get(path) ?? null;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
async list(filter?: KnowledgeFileFilter): Promise<KnowledgeFile[]> {
|
|
17
|
+
let results = Array.from(this.files.values());
|
|
18
|
+
if (filter?.type) {
|
|
19
|
+
results = results.filter((f) => f.type === filter.type);
|
|
20
|
+
}
|
|
21
|
+
if (filter?.quality) {
|
|
22
|
+
results = results.filter((f) => f.quality === filter.quality);
|
|
23
|
+
}
|
|
24
|
+
if (filter?.source) {
|
|
25
|
+
results = results.filter((f) => f.source === filter.source);
|
|
26
|
+
}
|
|
27
|
+
if (filter?.since) {
|
|
28
|
+
const since = filter.since;
|
|
29
|
+
results = results.filter((f) => f.ingested_at >= since);
|
|
30
|
+
}
|
|
31
|
+
return results;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async delete(path: string): Promise<boolean> {
|
|
35
|
+
return this.files.delete(path);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async count(): Promise<number> {
|
|
39
|
+
return this.files.size;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async saveGraph(graph: KnowledgeGraph): Promise<void> {
|
|
43
|
+
this.graph = graph;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async loadGraph(): Promise<KnowledgeGraph | null> {
|
|
47
|
+
return this.graph;
|
|
48
|
+
}
|
|
49
|
+
}
|
package/src/types.ts
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { Entity, GraphEdge, KnowledgeFile } from "@sourcepress/core";
|
|
2
|
+
|
|
3
|
+
export interface ExtractionResult {
|
|
4
|
+
entities: ExtractedEntity[];
|
|
5
|
+
relations: ExtractedRelation[];
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface ExtractedEntity extends Entity {
|
|
9
|
+
aliases: string[];
|
|
10
|
+
confidence: number;
|
|
11
|
+
source_file: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface ExtractedRelation {
|
|
15
|
+
from_entity: string;
|
|
16
|
+
to_entity: string;
|
|
17
|
+
relation_type: string;
|
|
18
|
+
confidence: number;
|
|
19
|
+
evidence: string;
|
|
20
|
+
source_file: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface EntityCluster {
|
|
24
|
+
id: string;
|
|
25
|
+
name: string;
|
|
26
|
+
entities: string[];
|
|
27
|
+
coherence_score: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface KnowledgeGraph {
|
|
31
|
+
entities: Map<string, ExtractedEntity>;
|
|
32
|
+
relations: ExtractedRelation[];
|
|
33
|
+
clusters: EntityCluster[];
|
|
34
|
+
built_at: string;
|
|
35
|
+
file_count: number;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface GraphQueryResult {
|
|
39
|
+
entity: ExtractedEntity;
|
|
40
|
+
relations: ExtractedRelation[];
|
|
41
|
+
related_entities: ExtractedEntity[];
|
|
42
|
+
files: string[];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface KnowledgeGap {
|
|
46
|
+
entity_name: string;
|
|
47
|
+
entity_type: string;
|
|
48
|
+
knowledge_file_count: number;
|
|
49
|
+
content_file_count: number;
|
|
50
|
+
reason: string;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface StaleContent {
|
|
54
|
+
content_path: string;
|
|
55
|
+
generated_at: string;
|
|
56
|
+
newest_source_change: string;
|
|
57
|
+
stale_sources: string[];
|
|
58
|
+
reason: string;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface KnowledgeStoreBackend {
|
|
62
|
+
store(file: KnowledgeFile): Promise<void>;
|
|
63
|
+
retrieve(path: string): Promise<KnowledgeFile | null>;
|
|
64
|
+
list(filter?: KnowledgeFileFilter): Promise<KnowledgeFile[]>;
|
|
65
|
+
delete(path: string): Promise<boolean>;
|
|
66
|
+
count(): Promise<number>;
|
|
67
|
+
saveGraph(graph: KnowledgeGraph): Promise<void>;
|
|
68
|
+
loadGraph(): Promise<KnowledgeGraph | null>;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface KnowledgeFileFilter {
|
|
72
|
+
type?: string;
|
|
73
|
+
quality?: KnowledgeFile["quality"];
|
|
74
|
+
source?: KnowledgeFile["source"];
|
|
75
|
+
since?: string;
|
|
76
|
+
}
|
package/tsconfig.json
ADDED
package/vitest.config.ts
ADDED