@odla-ai/kg 0.2.4 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-V3N2GRYU.js +73 -0
- package/dist/chunk-V3N2GRYU.js.map +1 -0
- package/dist/index.cjs +136 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +44 -250
- package/dist/index.d.ts +44 -250
- package/dist/index.js +126 -0
- package/dist/index.js.map +1 -1
- package/dist/sources-nJyD0JUP.d.cts +256 -0
- package/dist/sources-nJyD0JUP.d.ts +256 -0
- package/dist/temporal/index.cjs +1086 -0
- package/dist/temporal/index.cjs.map +1 -0
- package/dist/temporal/index.d.cts +690 -0
- package/dist/temporal/index.d.ts +690 -0
- package/dist/temporal/index.js +1004 -0
- package/dist/temporal/index.js.map +1 -0
- package/package.json +9 -2
package/dist/index.d.cts
CHANGED
|
@@ -1,123 +1,47 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
key: string;
|
|
23
|
-
attrs: Record<string, Value>;
|
|
24
|
-
}
|
|
25
|
-
/** A graph edge = an odla-db `link` from one node to another by forward label. */
|
|
26
|
-
interface GraphEdge {
|
|
27
|
-
from: {
|
|
28
|
-
type: string;
|
|
29
|
-
key: string;
|
|
1
|
+
import { O as Ontology, G as GraphFragment, M as MergeRule, P as Provider, S as SourceConnector, C as ConnectorContext, R as RawItem, a as SourceEntry, E as ExtractResult } from './sources-nJyD0JUP.cjs';
|
|
2
|
+
export { A as AttrSpec, b as AttrType, c as ConnectorJob, d as EdgeSpec, e as EntitySpec, f as ExtractCall, g as GraphEdge, h as GraphNode, L as LlmConfig, i as LlmEnv, j as MODEL_TIERS, N as NaturalKeys, k as ProviderFamily, l as SearchCall, U as Usage, W as WriteCall, m as addUsage, n as allJobs, o as createProvider, p as emptyUsage, q as enabledSources, r as fragmentToOps, s as getProvider, t as humanAttrsOf, u as jobsForHour, v as mergeRulesOf, w as naturalKeysOf, x as toSerializedSchema, y as writeFragment } from './sources-nJyD0JUP.cjs';
|
|
3
|
+
import { AdminDb, Value, Entity } from '@odla-ai/db';
|
|
4
|
+
import '@odla-ai/ai';
|
|
5
|
+
|
|
6
|
+
/** One entity type to export. */
|
|
7
|
+
interface ExportEntitySpec {
|
|
8
|
+
/** Link labels (forward or reverse, as queried in InstaQL) to pull as nested rows. */
|
|
9
|
+
links?: readonly string[];
|
|
10
|
+
/** Attr allowlist kept on nodes. Default: every scalar attr except `id` and links. */
|
|
11
|
+
attrs?: readonly string[];
|
|
12
|
+
/**
|
|
13
|
+
* Date window. Filtering is client-side by default; set `indexed: true`
|
|
14
|
+
* (only when the attr really is indexed) to also push a
|
|
15
|
+
* `where { [dateAttr]: { $gte: from } }` to the server — odla-db silently
|
|
16
|
+
* returns zero rows for `where` on unindexed attrs.
|
|
17
|
+
*/
|
|
18
|
+
window?: {
|
|
19
|
+
dateAttr: string;
|
|
20
|
+
from: string;
|
|
21
|
+
indexed?: boolean;
|
|
30
22
|
};
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
key: string;
|
|
34
|
-
};
|
|
35
|
-
label: string;
|
|
36
|
-
}
|
|
37
|
-
/** The single shape the graph writer consumes, whatever produced it. */
|
|
38
|
-
interface GraphFragment {
|
|
39
|
-
nodes: GraphNode[];
|
|
40
|
-
edges: GraphEdge[];
|
|
41
|
-
}
|
|
42
|
-
/** What an extractor turns a text RawItem into: a fragment plus a prose summary. */
|
|
43
|
-
interface ExtractResult {
|
|
44
|
-
fragment: GraphFragment;
|
|
45
|
-
summary: string;
|
|
23
|
+
/** Row cap for this type across the offset walk. Default 1000. */
|
|
24
|
+
limit?: number;
|
|
46
25
|
}
|
|
47
|
-
/**
|
|
48
|
-
interface
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
26
|
+
/** What {@link exportFragment} should read. */
|
|
27
|
+
interface ExportSpec {
|
|
28
|
+
entities: Record<string, ExportEntitySpec>;
|
|
29
|
+
/** Rows per page in the offset walk (1–1000). Default 200. */
|
|
30
|
+
pageSize?: number;
|
|
52
31
|
}
|
|
53
|
-
/** Create zeroed provider-agnostic usage totals for one pipeline run. */
|
|
54
|
-
declare function emptyUsage(): Usage;
|
|
55
|
-
/** Mutate an accumulator by adding call, input-token, and output-token totals. */
|
|
56
|
-
declare function addUsage(into: Usage, more: Usage): void;
|
|
57
|
-
|
|
58
|
-
/** node type → unique natural-key attribute name. */
|
|
59
|
-
type NaturalKeys = Record<string, string>;
|
|
60
32
|
/**
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
33
|
+
* Export (a window of) the graph as a `GraphFragment` — nodes and forward
|
|
34
|
+
* edges — ready for `extractEvents` and the rest of `@odla-ai/kg/temporal`.
|
|
35
|
+
*
|
|
36
|
+
* Each entity type is read with a stable offset walk (ordered by its natural
|
|
37
|
+
* key) carrying its requested link labels as nested relations. Date windows
|
|
38
|
+
* filter client-side; for attrs declared `indexed` the filter also runs
|
|
39
|
+
* server-side, and if that indexed filter ever exceeds the scan budget
|
|
40
|
+
* (`query_scan_limit`) the walk restarts without `where`.
|
|
41
|
+
* Nodes dedupe by `type:key` keeping the richer attr set; output ordering is
|
|
42
|
+
* deterministic (nodes by type/key, edges by from/label/to).
|
|
64
43
|
*/
|
|
65
|
-
declare function
|
|
66
|
-
/**
|
|
67
|
-
* Provenance-preserving write. Reads existing nodes by natural key (one batched
|
|
68
|
-
* query), merges scalar attrs per the ontology's rules, drops human-owned
|
|
69
|
-
* attrs, then writes merged node updates + all links in one atomic, idempotent
|
|
70
|
-
* transaction. `mutationId` must be deterministic so odla-db can deduplicate
|
|
71
|
-
* the write on retry; this does not make preceding reads or external effects
|
|
72
|
-
* exactly-once.
|
|
73
|
-
*/
|
|
74
|
-
declare function writeFragment(db: AdminDb, fragment: GraphFragment, ontology: Ontology, mutationId: string): Promise<{
|
|
75
|
-
txId: number;
|
|
76
|
-
ops: number;
|
|
77
|
-
}>;
|
|
78
|
-
|
|
79
|
-
/** How a scalar attribute is reconciled when a node is re-discovered. */
|
|
80
|
-
type MergeRule = "replace" | "prefer" | "earliest" | "latest";
|
|
81
|
-
type AttrType = SerializedAttr["type"];
|
|
82
|
-
interface AttrSpec {
|
|
83
|
-
type: AttrType;
|
|
84
|
-
unique?: boolean;
|
|
85
|
-
indexed?: boolean;
|
|
86
|
-
optional?: boolean;
|
|
87
|
-
/** The pipeline NEVER writes this attr (hand-edited in a UI); omitted from every update. */
|
|
88
|
-
human?: boolean;
|
|
89
|
-
/** Provenance rule for re-discovery. Default "replace". */
|
|
90
|
-
merge?: MergeRule;
|
|
91
|
-
/** Shown to the LLM when this attr appears in an extraction schema. */
|
|
92
|
-
desc?: string;
|
|
93
|
-
}
|
|
94
|
-
interface EntitySpec {
|
|
95
|
-
/** Unique attr used for Lookup upsert (MERGE key). Must be `unique` in attrs. */
|
|
96
|
-
naturalKey: string;
|
|
97
|
-
attrs: Record<string, AttrSpec>;
|
|
98
|
-
/** FTS fields for fuzzy resolution (P5). */
|
|
99
|
-
searchable?: string[];
|
|
100
|
-
}
|
|
101
|
-
interface EdgeSpec {
|
|
102
|
-
from: string;
|
|
103
|
-
to: string;
|
|
104
|
-
forwardLabel: string;
|
|
105
|
-
reverseLabel: string;
|
|
106
|
-
forwardHas?: Cardinality;
|
|
107
|
-
reverseHas?: Cardinality;
|
|
108
|
-
}
|
|
109
|
-
interface Ontology {
|
|
110
|
-
entities: Record<string, EntitySpec>;
|
|
111
|
-
edges: Record<string, EdgeSpec>;
|
|
112
|
-
}
|
|
113
|
-
/** node type → natural-key attr name (for the graph writer's Lookup refs). */
|
|
114
|
-
declare function naturalKeysOf(o: Ontology): NaturalKeys;
|
|
115
|
-
/** Attr name → merge rule, for every entity type (used by the provenance writer). */
|
|
116
|
-
declare function mergeRulesOf(o: Ontology): Record<string, Record<string, MergeRule>>;
|
|
117
|
-
/** Attr names the pipeline must never write, per entity type. */
|
|
118
|
-
declare function humanAttrsOf(o: Ontology): Record<string, Set<string>>;
|
|
119
|
-
/** Compile the ontology to an odla-db SerializedSchema to POST at /app/:id/schema. */
|
|
120
|
-
declare function toSerializedSchema(o: Ontology): SerializedSchema;
|
|
44
|
+
declare function exportFragment(db: AdminDb, ontology: Ontology, spec: ExportSpec): Promise<GraphFragment>;
|
|
121
45
|
|
|
122
46
|
/**
|
|
123
47
|
* Reconcile one scalar attribute when a node is re-discovered. Port of agnacl's
|
|
@@ -147,83 +71,6 @@ declare function resolveName(db: AdminDb, type: string, keyAttr: string, name: s
|
|
|
147
71
|
*/
|
|
148
72
|
declare function canonicalizeFragment(db: AdminDb, fragment: GraphFragment, ontology: Ontology): Promise<GraphFragment>;
|
|
149
73
|
|
|
150
|
-
/**
|
|
151
|
-
* The LLM seam, now a thin adapter over @odla-ai/ai (which was itself
|
|
152
|
-
* generalized from this package's original Provider). Three verbs stay:
|
|
153
|
-
* extract (forced structured tool call), search (web-search-enabled text),
|
|
154
|
-
* write (plain generation). Callers never see vendor or oracle types.
|
|
155
|
-
*/
|
|
156
|
-
type ProviderFamily = "claude" | "openai";
|
|
157
|
-
/** Role → model tiers per family — config as data. `cheap` is for gate-style
|
|
158
|
-
* calls, `extract` for structured extraction (and prose writing), `search`
|
|
159
|
-
* for web-search-enabled calls. */
|
|
160
|
-
declare const MODEL_TIERS: Record<ProviderFamily, {
|
|
161
|
-
cheap: string;
|
|
162
|
-
extract: string;
|
|
163
|
-
search: string;
|
|
164
|
-
}>;
|
|
165
|
-
/** A forced structured tool call: schema in, validated object out. */
|
|
166
|
-
interface ExtractCall {
|
|
167
|
-
system: string;
|
|
168
|
-
user: string;
|
|
169
|
-
tool: {
|
|
170
|
-
name: string;
|
|
171
|
-
description?: string;
|
|
172
|
-
parameters: Record<string, unknown>;
|
|
173
|
-
};
|
|
174
|
-
model?: string;
|
|
175
|
-
maxTokens?: number;
|
|
176
|
-
}
|
|
177
|
-
/** A web-search-enabled call: the model searches and returns text (caller tolerant-parses). */
|
|
178
|
-
interface SearchCall {
|
|
179
|
-
system: string;
|
|
180
|
-
user: string;
|
|
181
|
-
model?: string;
|
|
182
|
-
maxTokens?: number;
|
|
183
|
-
}
|
|
184
|
-
/** A plain text-generation call (no tools, no search) — e.g. digest prose. */
|
|
185
|
-
interface WriteCall {
|
|
186
|
-
system: string;
|
|
187
|
-
user: string;
|
|
188
|
-
model?: string;
|
|
189
|
-
maxTokens?: number;
|
|
190
|
-
}
|
|
191
|
-
interface Provider {
|
|
192
|
-
id: ProviderFamily;
|
|
193
|
-
/** The underlying @odla-ai/ai client — escape hatch for advanced use. */
|
|
194
|
-
ai: Ai;
|
|
195
|
-
extract<T = unknown>(c: ExtractCall): Promise<{
|
|
196
|
-
value: T;
|
|
197
|
-
usage: Usage;
|
|
198
|
-
}>;
|
|
199
|
-
search(c: SearchCall): Promise<{
|
|
200
|
-
text: string;
|
|
201
|
-
usage: Usage;
|
|
202
|
-
}>;
|
|
203
|
-
write(c: WriteCall): Promise<{
|
|
204
|
-
text: string;
|
|
205
|
-
usage: Usage;
|
|
206
|
-
}>;
|
|
207
|
-
}
|
|
208
|
-
interface LlmConfig {
|
|
209
|
-
provider: ProviderFamily;
|
|
210
|
-
anthropicKey?: string;
|
|
211
|
-
openaiKey?: string;
|
|
212
|
-
/** Inject a pre-built Ai (tests, custom catalog); keys are ignored when set. */
|
|
213
|
-
ai?: Ai;
|
|
214
|
-
}
|
|
215
|
-
/** The library primitive: an explicit config in, the three-verb seam out. */
|
|
216
|
-
declare function createProvider(cfg: LlmConfig): Provider;
|
|
217
|
-
/** The env vars the provider factory reads — a structural slice, not the
|
|
218
|
-
* worker Env, so any host (worker, script, test) can supply plain strings. */
|
|
219
|
-
interface LlmEnv {
|
|
220
|
-
LLM_PROVIDER?: string;
|
|
221
|
-
ANTHROPIC_API_KEY?: string;
|
|
222
|
-
OPENAI_API_KEY?: string;
|
|
223
|
-
}
|
|
224
|
-
/** Build the configured provider from host env, failing before use when its key is missing. */
|
|
225
|
-
declare function getProvider(env: LlmEnv): Provider;
|
|
226
|
-
|
|
227
74
|
/**
|
|
228
75
|
* Tolerant JSON extraction from LLM prose — web-search models return JSON inside
|
|
229
76
|
* explanatory text. Ported from agnacl services/enrichment.ts.
|
|
@@ -232,41 +79,6 @@ declare function extractJson<T = unknown>(text: string): T | null;
|
|
|
232
79
|
/** Tolerantly parse the first JSON-array-shaped span in model prose, or return an empty array. */
|
|
233
80
|
declare function extractJsonArray<T = unknown>(text: string): T[];
|
|
234
81
|
|
|
235
|
-
/** Ambient services a connector may use. Host-agnostic: secrets come through
|
|
236
|
-
* secret(), never a worker Env, so the library runs anywhere. */
|
|
237
|
-
interface ConnectorContext {
|
|
238
|
-
now: Date;
|
|
239
|
-
fetchText(url: string): Promise<string>;
|
|
240
|
-
secret(name: string): string | undefined;
|
|
241
|
-
ontology: Ontology;
|
|
242
|
-
/** The configured LLM provider (lazy — only web-search/discovery connectors need it). */
|
|
243
|
-
provider(): Provider;
|
|
244
|
-
}
|
|
245
|
-
/**
|
|
246
|
-
* The extensibility seam. A connector emits normalized RawItems — text-bearing
|
|
247
|
-
* (→ LLM extraction) or fragment-bearing (structured → straight to the writer).
|
|
248
|
-
* Adding a source = a new module implementing this + one registry line.
|
|
249
|
-
*/
|
|
250
|
-
interface SourceConnector<Cfg = unknown> {
|
|
251
|
-
id: string;
|
|
252
|
-
capabilities: {
|
|
253
|
-
fetch: boolean;
|
|
254
|
-
discover?: boolean;
|
|
255
|
-
structured: boolean;
|
|
256
|
-
};
|
|
257
|
-
parseConfig(raw: unknown): Cfg;
|
|
258
|
-
fetch(cfg: Cfg, ctx: ConnectorContext): Promise<RawItem[]>;
|
|
259
|
-
/** Web-search-style net-new discovery (P4). */
|
|
260
|
-
discover?(cfg: Cfg, ctx: ConnectorContext): Promise<RawItem[]>;
|
|
261
|
-
}
|
|
262
|
-
/** One unit of work for the ingest workflow: which connector + its config. */
|
|
263
|
-
interface ConnectorJob {
|
|
264
|
-
kind: string;
|
|
265
|
-
name: string;
|
|
266
|
-
watch?: string | null;
|
|
267
|
-
cfg: unknown;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
82
|
/** Resolve a built-in source connector by its stable registry id. */
|
|
271
83
|
declare function getConnector(id: string): SourceConnector | undefined;
|
|
272
84
|
/** List registered connector ids in deterministic insertion order. */
|
|
@@ -285,28 +97,6 @@ declare function makeContext(cfg: ContextConfig): ConnectorContext;
|
|
|
285
97
|
/** Fetch one URL → a text-bearing RawItem. Used both directly (/trigger) and as a connector. */
|
|
286
98
|
declare function fetchUrl(url: string): Promise<RawItem>;
|
|
287
99
|
|
|
288
|
-
/**
|
|
289
|
-
* Sources are DATA, not code. Each entry names a connector (`kind`) + its config
|
|
290
|
-
* + scheduling. Add a feed → add an entry; no redeploy of the pipeline. P5 moves
|
|
291
|
-
* this into KV behind a /config admin endpoint so edits don't need a deploy at all.
|
|
292
|
-
*/
|
|
293
|
-
interface SourceEntry {
|
|
294
|
-
kind: string;
|
|
295
|
-
name: string;
|
|
296
|
-
enabled?: boolean;
|
|
297
|
-
cadenceHours?: number;
|
|
298
|
-
watch?: string | null;
|
|
299
|
-
url?: string;
|
|
300
|
-
query?: string;
|
|
301
|
-
[key: string]: unknown;
|
|
302
|
-
}
|
|
303
|
-
/** Return source entries not explicitly disabled, preserving configured order. */
|
|
304
|
-
declare function enabledSources(sources: SourceEntry[]): SourceEntry[];
|
|
305
|
-
/** Enabled sources whose cadence lands on this UTC hour. */
|
|
306
|
-
declare function jobsForHour(sources: SourceEntry[], hour: number): ConnectorJob[];
|
|
307
|
-
/** Every enabled source, ignoring cadence (manual /trigger?kind=sources). */
|
|
308
|
-
declare function allJobs(sources: SourceEntry[]): ConnectorJob[];
|
|
309
|
-
|
|
310
100
|
/**
|
|
311
101
|
* The slice of a KV store the pipeline needs. Cloudflare's KVNamespace
|
|
312
102
|
* satisfies it structurally; tests pass an in-memory map. Keeping the library
|
|
@@ -338,6 +128,10 @@ declare class Store {
|
|
|
338
128
|
/** Runtime source config (edit without redeploy). null = fall back to the static SOURCES. */
|
|
339
129
|
getConfigSources(): Promise<SourceEntry[] | null>;
|
|
340
130
|
setConfigSources(list: SourceEntry[]): Promise<void>;
|
|
131
|
+
/** Cache the latest temporal report (`temporal:latest`) plus a dated history copy. */
|
|
132
|
+
setTemporalReport(report: unknown, date?: string): Promise<void>;
|
|
133
|
+
/** The cached latest temporal report, or null when absent/unparseable. */
|
|
134
|
+
getTemporalReport<T = unknown>(): Promise<T | null>;
|
|
341
135
|
}
|
|
342
136
|
|
|
343
137
|
/**
|
|
@@ -442,4 +236,4 @@ declare function htmlToText(html: string, maxChars?: number): string;
|
|
|
442
236
|
/** Fetch a URL and return readable text. Best-effort: returns "" on any failure. */
|
|
443
237
|
declare function fetchText(url: string, maxChars?: number): Promise<string>;
|
|
444
238
|
|
|
445
|
-
export {
|
|
239
|
+
export { COMMIT_STEP_CONFIG, ConnectorContext, type ContextConfig, type ExportEntitySpec, type ExportSpec, ExtractResult, GraphFragment, IO_STEP_CONFIG, type KVLike, LLM_STEP_CONFIG, MergeRule, Ontology, type PipelineDeps, Provider, RawItem, SourceConnector, SourceEntry, type StepConfig, type StepLike, Store, asStepLike, canonicalizeFragment, connectorIds, exportFragment, extractJson, extractJsonArray, fetchText, fetchUrl, getConnector, htmlToText, ingestFresh, isLikelyPersonName, makeContext, mergeAttrs, mergeScalar, normalizeUrl, resilientStep, resolveName, searchEntities, slugify, urlHash };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,123 +1,47 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
key: string;
|
|
23
|
-
attrs: Record<string, Value>;
|
|
24
|
-
}
|
|
25
|
-
/** A graph edge = an odla-db `link` from one node to another by forward label. */
|
|
26
|
-
interface GraphEdge {
|
|
27
|
-
from: {
|
|
28
|
-
type: string;
|
|
29
|
-
key: string;
|
|
1
|
+
import { O as Ontology, G as GraphFragment, M as MergeRule, P as Provider, S as SourceConnector, C as ConnectorContext, R as RawItem, a as SourceEntry, E as ExtractResult } from './sources-nJyD0JUP.js';
|
|
2
|
+
export { A as AttrSpec, b as AttrType, c as ConnectorJob, d as EdgeSpec, e as EntitySpec, f as ExtractCall, g as GraphEdge, h as GraphNode, L as LlmConfig, i as LlmEnv, j as MODEL_TIERS, N as NaturalKeys, k as ProviderFamily, l as SearchCall, U as Usage, W as WriteCall, m as addUsage, n as allJobs, o as createProvider, p as emptyUsage, q as enabledSources, r as fragmentToOps, s as getProvider, t as humanAttrsOf, u as jobsForHour, v as mergeRulesOf, w as naturalKeysOf, x as toSerializedSchema, y as writeFragment } from './sources-nJyD0JUP.js';
|
|
3
|
+
import { AdminDb, Value, Entity } from '@odla-ai/db';
|
|
4
|
+
import '@odla-ai/ai';
|
|
5
|
+
|
|
6
|
+
/** One entity type to export. */
|
|
7
|
+
interface ExportEntitySpec {
|
|
8
|
+
/** Link labels (forward or reverse, as queried in InstaQL) to pull as nested rows. */
|
|
9
|
+
links?: readonly string[];
|
|
10
|
+
/** Attr allowlist kept on nodes. Default: every scalar attr except `id` and links. */
|
|
11
|
+
attrs?: readonly string[];
|
|
12
|
+
/**
|
|
13
|
+
* Date window. Filtering is client-side by default; set `indexed: true`
|
|
14
|
+
* (only when the attr really is indexed) to also push a
|
|
15
|
+
* `where { [dateAttr]: { $gte: from } }` to the server — odla-db silently
|
|
16
|
+
* returns zero rows for `where` on unindexed attrs.
|
|
17
|
+
*/
|
|
18
|
+
window?: {
|
|
19
|
+
dateAttr: string;
|
|
20
|
+
from: string;
|
|
21
|
+
indexed?: boolean;
|
|
30
22
|
};
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
key: string;
|
|
34
|
-
};
|
|
35
|
-
label: string;
|
|
36
|
-
}
|
|
37
|
-
/** The single shape the graph writer consumes, whatever produced it. */
|
|
38
|
-
interface GraphFragment {
|
|
39
|
-
nodes: GraphNode[];
|
|
40
|
-
edges: GraphEdge[];
|
|
41
|
-
}
|
|
42
|
-
/** What an extractor turns a text RawItem into: a fragment plus a prose summary. */
|
|
43
|
-
interface ExtractResult {
|
|
44
|
-
fragment: GraphFragment;
|
|
45
|
-
summary: string;
|
|
23
|
+
/** Row cap for this type across the offset walk. Default 1000. */
|
|
24
|
+
limit?: number;
|
|
46
25
|
}
|
|
47
|
-
/**
|
|
48
|
-
interface
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
26
|
+
/** What {@link exportFragment} should read. */
|
|
27
|
+
interface ExportSpec {
|
|
28
|
+
entities: Record<string, ExportEntitySpec>;
|
|
29
|
+
/** Rows per page in the offset walk (1–1000). Default 200. */
|
|
30
|
+
pageSize?: number;
|
|
52
31
|
}
|
|
53
|
-
/** Create zeroed provider-agnostic usage totals for one pipeline run. */
|
|
54
|
-
declare function emptyUsage(): Usage;
|
|
55
|
-
/** Mutate an accumulator by adding call, input-token, and output-token totals. */
|
|
56
|
-
declare function addUsage(into: Usage, more: Usage): void;
|
|
57
|
-
|
|
58
|
-
/** node type → unique natural-key attribute name. */
|
|
59
|
-
type NaturalKeys = Record<string, string>;
|
|
60
32
|
/**
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
33
|
+
* Export (a window of) the graph as a `GraphFragment` — nodes and forward
|
|
34
|
+
* edges — ready for `extractEvents` and the rest of `@odla-ai/kg/temporal`.
|
|
35
|
+
*
|
|
36
|
+
* Each entity type is read with a stable offset walk (ordered by its natural
|
|
37
|
+
* key) carrying its requested link labels as nested relations. Date windows
|
|
38
|
+
* filter client-side; for attrs declared `indexed` the filter also runs
|
|
39
|
+
* server-side, and if that indexed filter ever exceeds the scan budget
|
|
40
|
+
* (`query_scan_limit`) the walk restarts without `where`.
|
|
41
|
+
* Nodes dedupe by `type:key` keeping the richer attr set; output ordering is
|
|
42
|
+
* deterministic (nodes by type/key, edges by from/label/to).
|
|
64
43
|
*/
|
|
65
|
-
declare function
|
|
66
|
-
/**
|
|
67
|
-
* Provenance-preserving write. Reads existing nodes by natural key (one batched
|
|
68
|
-
* query), merges scalar attrs per the ontology's rules, drops human-owned
|
|
69
|
-
* attrs, then writes merged node updates + all links in one atomic, idempotent
|
|
70
|
-
* transaction. `mutationId` must be deterministic so odla-db can deduplicate
|
|
71
|
-
* the write on retry; this does not make preceding reads or external effects
|
|
72
|
-
* exactly-once.
|
|
73
|
-
*/
|
|
74
|
-
declare function writeFragment(db: AdminDb, fragment: GraphFragment, ontology: Ontology, mutationId: string): Promise<{
|
|
75
|
-
txId: number;
|
|
76
|
-
ops: number;
|
|
77
|
-
}>;
|
|
78
|
-
|
|
79
|
-
/** How a scalar attribute is reconciled when a node is re-discovered. */
|
|
80
|
-
type MergeRule = "replace" | "prefer" | "earliest" | "latest";
|
|
81
|
-
type AttrType = SerializedAttr["type"];
|
|
82
|
-
interface AttrSpec {
|
|
83
|
-
type: AttrType;
|
|
84
|
-
unique?: boolean;
|
|
85
|
-
indexed?: boolean;
|
|
86
|
-
optional?: boolean;
|
|
87
|
-
/** The pipeline NEVER writes this attr (hand-edited in a UI); omitted from every update. */
|
|
88
|
-
human?: boolean;
|
|
89
|
-
/** Provenance rule for re-discovery. Default "replace". */
|
|
90
|
-
merge?: MergeRule;
|
|
91
|
-
/** Shown to the LLM when this attr appears in an extraction schema. */
|
|
92
|
-
desc?: string;
|
|
93
|
-
}
|
|
94
|
-
interface EntitySpec {
|
|
95
|
-
/** Unique attr used for Lookup upsert (MERGE key). Must be `unique` in attrs. */
|
|
96
|
-
naturalKey: string;
|
|
97
|
-
attrs: Record<string, AttrSpec>;
|
|
98
|
-
/** FTS fields for fuzzy resolution (P5). */
|
|
99
|
-
searchable?: string[];
|
|
100
|
-
}
|
|
101
|
-
interface EdgeSpec {
|
|
102
|
-
from: string;
|
|
103
|
-
to: string;
|
|
104
|
-
forwardLabel: string;
|
|
105
|
-
reverseLabel: string;
|
|
106
|
-
forwardHas?: Cardinality;
|
|
107
|
-
reverseHas?: Cardinality;
|
|
108
|
-
}
|
|
109
|
-
interface Ontology {
|
|
110
|
-
entities: Record<string, EntitySpec>;
|
|
111
|
-
edges: Record<string, EdgeSpec>;
|
|
112
|
-
}
|
|
113
|
-
/** node type → natural-key attr name (for the graph writer's Lookup refs). */
|
|
114
|
-
declare function naturalKeysOf(o: Ontology): NaturalKeys;
|
|
115
|
-
/** Attr name → merge rule, for every entity type (used by the provenance writer). */
|
|
116
|
-
declare function mergeRulesOf(o: Ontology): Record<string, Record<string, MergeRule>>;
|
|
117
|
-
/** Attr names the pipeline must never write, per entity type. */
|
|
118
|
-
declare function humanAttrsOf(o: Ontology): Record<string, Set<string>>;
|
|
119
|
-
/** Compile the ontology to an odla-db SerializedSchema to POST at /app/:id/schema. */
|
|
120
|
-
declare function toSerializedSchema(o: Ontology): SerializedSchema;
|
|
44
|
+
declare function exportFragment(db: AdminDb, ontology: Ontology, spec: ExportSpec): Promise<GraphFragment>;
|
|
121
45
|
|
|
122
46
|
/**
|
|
123
47
|
* Reconcile one scalar attribute when a node is re-discovered. Port of agnacl's
|
|
@@ -147,83 +71,6 @@ declare function resolveName(db: AdminDb, type: string, keyAttr: string, name: s
|
|
|
147
71
|
*/
|
|
148
72
|
declare function canonicalizeFragment(db: AdminDb, fragment: GraphFragment, ontology: Ontology): Promise<GraphFragment>;
|
|
149
73
|
|
|
150
|
-
/**
|
|
151
|
-
* The LLM seam, now a thin adapter over @odla-ai/ai (which was itself
|
|
152
|
-
* generalized from this package's original Provider). Three verbs stay:
|
|
153
|
-
* extract (forced structured tool call), search (web-search-enabled text),
|
|
154
|
-
* write (plain generation). Callers never see vendor or oracle types.
|
|
155
|
-
*/
|
|
156
|
-
type ProviderFamily = "claude" | "openai";
|
|
157
|
-
/** Role → model tiers per family — config as data. `cheap` is for gate-style
|
|
158
|
-
* calls, `extract` for structured extraction (and prose writing), `search`
|
|
159
|
-
* for web-search-enabled calls. */
|
|
160
|
-
declare const MODEL_TIERS: Record<ProviderFamily, {
|
|
161
|
-
cheap: string;
|
|
162
|
-
extract: string;
|
|
163
|
-
search: string;
|
|
164
|
-
}>;
|
|
165
|
-
/** A forced structured tool call: schema in, validated object out. */
|
|
166
|
-
interface ExtractCall {
|
|
167
|
-
system: string;
|
|
168
|
-
user: string;
|
|
169
|
-
tool: {
|
|
170
|
-
name: string;
|
|
171
|
-
description?: string;
|
|
172
|
-
parameters: Record<string, unknown>;
|
|
173
|
-
};
|
|
174
|
-
model?: string;
|
|
175
|
-
maxTokens?: number;
|
|
176
|
-
}
|
|
177
|
-
/** A web-search-enabled call: the model searches and returns text (caller tolerant-parses). */
|
|
178
|
-
interface SearchCall {
|
|
179
|
-
system: string;
|
|
180
|
-
user: string;
|
|
181
|
-
model?: string;
|
|
182
|
-
maxTokens?: number;
|
|
183
|
-
}
|
|
184
|
-
/** A plain text-generation call (no tools, no search) — e.g. digest prose. */
|
|
185
|
-
interface WriteCall {
|
|
186
|
-
system: string;
|
|
187
|
-
user: string;
|
|
188
|
-
model?: string;
|
|
189
|
-
maxTokens?: number;
|
|
190
|
-
}
|
|
191
|
-
interface Provider {
|
|
192
|
-
id: ProviderFamily;
|
|
193
|
-
/** The underlying @odla-ai/ai client — escape hatch for advanced use. */
|
|
194
|
-
ai: Ai;
|
|
195
|
-
extract<T = unknown>(c: ExtractCall): Promise<{
|
|
196
|
-
value: T;
|
|
197
|
-
usage: Usage;
|
|
198
|
-
}>;
|
|
199
|
-
search(c: SearchCall): Promise<{
|
|
200
|
-
text: string;
|
|
201
|
-
usage: Usage;
|
|
202
|
-
}>;
|
|
203
|
-
write(c: WriteCall): Promise<{
|
|
204
|
-
text: string;
|
|
205
|
-
usage: Usage;
|
|
206
|
-
}>;
|
|
207
|
-
}
|
|
208
|
-
interface LlmConfig {
|
|
209
|
-
provider: ProviderFamily;
|
|
210
|
-
anthropicKey?: string;
|
|
211
|
-
openaiKey?: string;
|
|
212
|
-
/** Inject a pre-built Ai (tests, custom catalog); keys are ignored when set. */
|
|
213
|
-
ai?: Ai;
|
|
214
|
-
}
|
|
215
|
-
/** The library primitive: an explicit config in, the three-verb seam out. */
|
|
216
|
-
declare function createProvider(cfg: LlmConfig): Provider;
|
|
217
|
-
/** The env vars the provider factory reads — a structural slice, not the
|
|
218
|
-
* worker Env, so any host (worker, script, test) can supply plain strings. */
|
|
219
|
-
interface LlmEnv {
|
|
220
|
-
LLM_PROVIDER?: string;
|
|
221
|
-
ANTHROPIC_API_KEY?: string;
|
|
222
|
-
OPENAI_API_KEY?: string;
|
|
223
|
-
}
|
|
224
|
-
/** Build the configured provider from host env, failing before use when its key is missing. */
|
|
225
|
-
declare function getProvider(env: LlmEnv): Provider;
|
|
226
|
-
|
|
227
74
|
/**
|
|
228
75
|
* Tolerant JSON extraction from LLM prose — web-search models return JSON inside
|
|
229
76
|
* explanatory text. Ported from agnacl services/enrichment.ts.
|
|
@@ -232,41 +79,6 @@ declare function extractJson<T = unknown>(text: string): T | null;
|
|
|
232
79
|
/** Tolerantly parse the first JSON-array-shaped span in model prose, or return an empty array. */
|
|
233
80
|
declare function extractJsonArray<T = unknown>(text: string): T[];
|
|
234
81
|
|
|
235
|
-
/** Ambient services a connector may use. Host-agnostic: secrets come through
|
|
236
|
-
* secret(), never a worker Env, so the library runs anywhere. */
|
|
237
|
-
interface ConnectorContext {
|
|
238
|
-
now: Date;
|
|
239
|
-
fetchText(url: string): Promise<string>;
|
|
240
|
-
secret(name: string): string | undefined;
|
|
241
|
-
ontology: Ontology;
|
|
242
|
-
/** The configured LLM provider (lazy — only web-search/discovery connectors need it). */
|
|
243
|
-
provider(): Provider;
|
|
244
|
-
}
|
|
245
|
-
/**
|
|
246
|
-
* The extensibility seam. A connector emits normalized RawItems — text-bearing
|
|
247
|
-
* (→ LLM extraction) or fragment-bearing (structured → straight to the writer).
|
|
248
|
-
* Adding a source = a new module implementing this + one registry line.
|
|
249
|
-
*/
|
|
250
|
-
interface SourceConnector<Cfg = unknown> {
|
|
251
|
-
id: string;
|
|
252
|
-
capabilities: {
|
|
253
|
-
fetch: boolean;
|
|
254
|
-
discover?: boolean;
|
|
255
|
-
structured: boolean;
|
|
256
|
-
};
|
|
257
|
-
parseConfig(raw: unknown): Cfg;
|
|
258
|
-
fetch(cfg: Cfg, ctx: ConnectorContext): Promise<RawItem[]>;
|
|
259
|
-
/** Web-search-style net-new discovery (P4). */
|
|
260
|
-
discover?(cfg: Cfg, ctx: ConnectorContext): Promise<RawItem[]>;
|
|
261
|
-
}
|
|
262
|
-
/** One unit of work for the ingest workflow: which connector + its config. */
|
|
263
|
-
interface ConnectorJob {
|
|
264
|
-
kind: string;
|
|
265
|
-
name: string;
|
|
266
|
-
watch?: string | null;
|
|
267
|
-
cfg: unknown;
|
|
268
|
-
}
|
|
269
|
-
|
|
270
82
|
/** Resolve a built-in source connector by its stable registry id. */
|
|
271
83
|
declare function getConnector(id: string): SourceConnector | undefined;
|
|
272
84
|
/** List registered connector ids in deterministic insertion order. */
|
|
@@ -285,28 +97,6 @@ declare function makeContext(cfg: ContextConfig): ConnectorContext;
|
|
|
285
97
|
/** Fetch one URL → a text-bearing RawItem. Used both directly (/trigger) and as a connector. */
|
|
286
98
|
declare function fetchUrl(url: string): Promise<RawItem>;
|
|
287
99
|
|
|
288
|
-
/**
|
|
289
|
-
* Sources are DATA, not code. Each entry names a connector (`kind`) + its config
|
|
290
|
-
* + scheduling. Add a feed → add an entry; no redeploy of the pipeline. P5 moves
|
|
291
|
-
* this into KV behind a /config admin endpoint so edits don't need a deploy at all.
|
|
292
|
-
*/
|
|
293
|
-
interface SourceEntry {
|
|
294
|
-
kind: string;
|
|
295
|
-
name: string;
|
|
296
|
-
enabled?: boolean;
|
|
297
|
-
cadenceHours?: number;
|
|
298
|
-
watch?: string | null;
|
|
299
|
-
url?: string;
|
|
300
|
-
query?: string;
|
|
301
|
-
[key: string]: unknown;
|
|
302
|
-
}
|
|
303
|
-
/** Return source entries not explicitly disabled, preserving configured order. */
|
|
304
|
-
declare function enabledSources(sources: SourceEntry[]): SourceEntry[];
|
|
305
|
-
/** Enabled sources whose cadence lands on this UTC hour. */
|
|
306
|
-
declare function jobsForHour(sources: SourceEntry[], hour: number): ConnectorJob[];
|
|
307
|
-
/** Every enabled source, ignoring cadence (manual /trigger?kind=sources). */
|
|
308
|
-
declare function allJobs(sources: SourceEntry[]): ConnectorJob[];
|
|
309
|
-
|
|
310
100
|
/**
|
|
311
101
|
* The slice of a KV store the pipeline needs. Cloudflare's KVNamespace
|
|
312
102
|
* satisfies it structurally; tests pass an in-memory map. Keeping the library
|
|
@@ -338,6 +128,10 @@ declare class Store {
|
|
|
338
128
|
/** Runtime source config (edit without redeploy). null = fall back to the static SOURCES. */
|
|
339
129
|
getConfigSources(): Promise<SourceEntry[] | null>;
|
|
340
130
|
setConfigSources(list: SourceEntry[]): Promise<void>;
|
|
131
|
+
/** Cache the latest temporal report (`temporal:latest`) plus a dated history copy. */
|
|
132
|
+
setTemporalReport(report: unknown, date?: string): Promise<void>;
|
|
133
|
+
/** The cached latest temporal report, or null when absent/unparseable. */
|
|
134
|
+
getTemporalReport<T = unknown>(): Promise<T | null>;
|
|
341
135
|
}
|
|
342
136
|
|
|
343
137
|
/**
|
|
@@ -442,4 +236,4 @@ declare function htmlToText(html: string, maxChars?: number): string;
|
|
|
442
236
|
/** Fetch a URL and return readable text. Best-effort: returns "" on any failure. */
|
|
443
237
|
declare function fetchText(url: string, maxChars?: number): Promise<string>;
|
|
444
238
|
|
|
445
|
-
export {
|
|
239
|
+
export { COMMIT_STEP_CONFIG, ConnectorContext, type ContextConfig, type ExportEntitySpec, type ExportSpec, ExtractResult, GraphFragment, IO_STEP_CONFIG, type KVLike, LLM_STEP_CONFIG, MergeRule, Ontology, type PipelineDeps, Provider, RawItem, SourceConnector, SourceEntry, type StepConfig, type StepLike, Store, asStepLike, canonicalizeFragment, connectorIds, exportFragment, extractJson, extractJsonArray, fetchText, fetchUrl, getConnector, htmlToText, ingestFresh, isLikelyPersonName, makeContext, mergeAttrs, mergeScalar, normalizeUrl, resilientStep, resolveName, searchEntities, slugify, urlHash };
|