@f0rbit/corpus 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/backend/cloudflare.d.ts.map +1 -1
- package/dist/backend/cloudflare.js +140 -23
- package/dist/backend/file.d.ts.map +1 -1
- package/dist/backend/file.js +47 -48
- package/dist/backend/layered.d.ts.map +1 -1
- package/dist/backend/layered.js +67 -19
- package/dist/backend/memory.d.ts +2 -1
- package/dist/backend/memory.d.ts.map +1 -1
- package/dist/backend/memory.js +29 -43
- package/dist/corpus.d.ts +11 -0
- package/dist/corpus.d.ts.map +1 -1
- package/dist/corpus.js +52 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/observations/client.d.ts +12 -0
- package/dist/observations/client.d.ts.map +1 -0
- package/dist/observations/client.js +115 -0
- package/dist/observations/index.d.ts +12 -0
- package/dist/observations/index.d.ts.map +1 -0
- package/dist/observations/index.js +11 -0
- package/dist/observations/schema.d.ts +267 -0
- package/dist/observations/schema.d.ts.map +1 -0
- package/dist/observations/schema.js +55 -0
- package/dist/observations/storage.d.ts +75 -0
- package/dist/observations/storage.d.ts.map +1 -0
- package/dist/observations/storage.js +137 -0
- package/dist/observations/types.d.ts +219 -0
- package/dist/observations/types.d.ts.map +1 -0
- package/dist/observations/types.js +40 -0
- package/dist/observations/utils.d.ts +183 -0
- package/dist/observations/utils.d.ts.map +1 -0
- package/dist/observations/utils.js +272 -0
- package/dist/sst.d.ts +1 -1
- package/dist/sst.d.ts.map +1 -1
- package/dist/sst.js +20 -0
- package/dist/types.d.ts +61 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/utils.d.ts +38 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +84 -0
- package/package.json +71 -67
- package/dist/codecs.d.ts +0 -8
- package/dist/codecs.d.ts.map +0 -1
- package/dist/codecs.js +0 -6
- package/dist/core.d.ts +0 -9
- package/dist/core.d.ts.map +0 -1
- package/dist/core.js +0 -7
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../observations/storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AAEnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE5E;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;IACxB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAA;AAED;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,+CAA+C;IAC/C,OAAO,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,CAAA;IAE9E,kDAAkD;IAClD,OAAO,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,WAAW,CAAC,CAAC,CAAA;IAE5E,wCAAwC;IACxC,UAAU,EAAE,CAAC,IAAI,CAAC,EAAE,gBAAgB,KAAK,aAAa,CAAC,cAAc,CAAC,CAAA;IAEtE,uEAAuE;IACvE,UAAU,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAA;IAEjE,0DAA0D;IAC1D,gBAAgB,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAA;CAC7G,CAAA;AA4BD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,cAAc,GAAG,WAAW,CAKnE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,cAAc,GAAG,eAAe,CAEhE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,EAAE,EAAE,MAAM,EACV,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE;IACJ,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,IAAI,CAAA;IAClB,YAAY,CAAC,EAAE,eAAe,EAAE,CAAA;CACjC,GACA,cAAc,CAgBhB;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,cAAc,EAAE,EACtB,IAAI,GAAE,gBAAqB,GAC1B,cAAc,EAAE,CAsClB;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,OAAO,EAAE,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC,CAAA;IACxC,OAAO,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAClD,OAAO,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAA;IACvD,OAAO,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,UAAU,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAC7C,CAAA;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,gBAAgB,GAAG,mBAAmB,CAoCvF"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module ObservationsStorage
|
|
3
|
+
* @description Raw storage interface and row conversion utilities for observations.
|
|
4
|
+
*/
|
|
5
|
+
import { ok } from '../types';
|
|
6
|
+
/**
|
|
7
|
+
* Extract common fields from an observation row (everything except content).
|
|
8
|
+
* Used internally by row_to_observation and row_to_meta.
|
|
9
|
+
*/
|
|
10
|
+
function row_to_base(row) {
|
|
11
|
+
return {
|
|
12
|
+
id: row.id,
|
|
13
|
+
type: row.type,
|
|
14
|
+
source: {
|
|
15
|
+
store_id: row.source_store_id,
|
|
16
|
+
version: row.source_version,
|
|
17
|
+
...(row.source_path && { path: row.source_path }),
|
|
18
|
+
...(row.source_span_start && row.source_span_end && {
|
|
19
|
+
span: {
|
|
20
|
+
start: parseInt(row.source_span_start, 10),
|
|
21
|
+
end: parseInt(row.source_span_end, 10)
|
|
22
|
+
}
|
|
23
|
+
})
|
|
24
|
+
},
|
|
25
|
+
...(row.confidence !== null && { confidence: row.confidence }),
|
|
26
|
+
...(row.observed_at && { observed_at: new Date(row.observed_at) }),
|
|
27
|
+
created_at: new Date(row.created_at),
|
|
28
|
+
...(row.derived_from && { derived_from: JSON.parse(row.derived_from) })
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Convert a storage row to an Observation (includes content).
|
|
33
|
+
*/
|
|
34
|
+
export function row_to_observation(row) {
|
|
35
|
+
return {
|
|
36
|
+
...row_to_base(row),
|
|
37
|
+
content: JSON.parse(row.content)
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Convert a storage row to ObservationMeta (excludes content).
|
|
42
|
+
*/
|
|
43
|
+
export function row_to_meta(row) {
|
|
44
|
+
return row_to_base(row);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Create an ObservationRow from put options.
|
|
48
|
+
*/
|
|
49
|
+
export function create_observation_row(id, type_name, source, content, opts) {
|
|
50
|
+
const now = new Date();
|
|
51
|
+
return {
|
|
52
|
+
id,
|
|
53
|
+
type: type_name,
|
|
54
|
+
source_store_id: source.store_id,
|
|
55
|
+
source_version: source.version,
|
|
56
|
+
source_path: source.path ?? null,
|
|
57
|
+
source_span_start: source.span?.start?.toString() ?? null,
|
|
58
|
+
source_span_end: source.span?.end?.toString() ?? null,
|
|
59
|
+
content: JSON.stringify(content),
|
|
60
|
+
confidence: opts.confidence ?? null,
|
|
61
|
+
observed_at: opts.observed_at?.toISOString() ?? null,
|
|
62
|
+
created_at: now.toISOString(),
|
|
63
|
+
derived_from: opts.derived_from ? JSON.stringify(opts.derived_from) : null
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Filter and sort observation rows based on query options.
|
|
68
|
+
* Used by in-memory storage implementations (memory backend, file backend).
|
|
69
|
+
*/
|
|
70
|
+
export function filter_observation_rows(rows, opts = {}) {
|
|
71
|
+
let filtered = rows;
|
|
72
|
+
if (opts.type) {
|
|
73
|
+
const types = Array.isArray(opts.type) ? opts.type : [opts.type];
|
|
74
|
+
filtered = filtered.filter(r => types.includes(r.type));
|
|
75
|
+
}
|
|
76
|
+
if (opts.source_store_id) {
|
|
77
|
+
filtered = filtered.filter(r => r.source_store_id === opts.source_store_id);
|
|
78
|
+
}
|
|
79
|
+
if (opts.source_version) {
|
|
80
|
+
filtered = filtered.filter(r => r.source_version === opts.source_version);
|
|
81
|
+
}
|
|
82
|
+
if (opts.source_prefix) {
|
|
83
|
+
filtered = filtered.filter(r => r.source_version.startsWith(opts.source_prefix));
|
|
84
|
+
}
|
|
85
|
+
if (opts.created_after) {
|
|
86
|
+
filtered = filtered.filter(r => r.created_at > opts.created_after);
|
|
87
|
+
}
|
|
88
|
+
if (opts.created_before) {
|
|
89
|
+
filtered = filtered.filter(r => r.created_at < opts.created_before);
|
|
90
|
+
}
|
|
91
|
+
if (opts.observed_after) {
|
|
92
|
+
filtered = filtered.filter(r => r.observed_at && r.observed_at > opts.observed_after);
|
|
93
|
+
}
|
|
94
|
+
if (opts.observed_before) {
|
|
95
|
+
filtered = filtered.filter(r => r.observed_at && r.observed_at < opts.observed_before);
|
|
96
|
+
}
|
|
97
|
+
filtered.sort((a, b) => b.created_at.localeCompare(a.created_at));
|
|
98
|
+
if (opts.limit) {
|
|
99
|
+
filtered = filtered.slice(0, opts.limit);
|
|
100
|
+
}
|
|
101
|
+
return filtered;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Create an ObservationsStorage from simple CRUD operations.
|
|
105
|
+
* Used by memory and file backends.
|
|
106
|
+
*/
|
|
107
|
+
export function create_observations_storage(crud) {
|
|
108
|
+
return {
|
|
109
|
+
async put_row(row) {
|
|
110
|
+
await crud.add_one(row);
|
|
111
|
+
return ok(row);
|
|
112
|
+
},
|
|
113
|
+
async get_row(id) {
|
|
114
|
+
const row = await crud.get_one(id);
|
|
115
|
+
return ok(row);
|
|
116
|
+
},
|
|
117
|
+
async *query_rows(opts = {}) {
|
|
118
|
+
const rows = filter_observation_rows(await crud.get_all(), opts);
|
|
119
|
+
for (const row of rows) {
|
|
120
|
+
yield row;
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
async delete_row(id) {
|
|
124
|
+
const deleted = await crud.remove_one(id);
|
|
125
|
+
return ok(deleted);
|
|
126
|
+
},
|
|
127
|
+
async delete_by_source(store_id, version, path) {
|
|
128
|
+
const rows = await crud.get_all();
|
|
129
|
+
const toKeep = rows.filter(r => !(r.source_store_id === store_id &&
|
|
130
|
+
r.source_version === version &&
|
|
131
|
+
(path === undefined || r.source_path === path)));
|
|
132
|
+
const deleted = rows.length - toKeep.length;
|
|
133
|
+
await crud.set_all(toKeep);
|
|
134
|
+
return ok(deleted);
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module ObservationTypes
|
|
3
|
+
* @description Type definitions for the observations feature.
|
|
4
|
+
*/
|
|
5
|
+
import type { ZodType } from 'zod';
|
|
6
|
+
/**
|
|
7
|
+
* Universal address to versioned content within a corpus store.
|
|
8
|
+
*
|
|
9
|
+
* A SnapshotPointer identifies a specific location within a versioned snapshot:
|
|
10
|
+
* - `store_id` + `version` - Required: which snapshot
|
|
11
|
+
* - `path` - Optional: JSONPath expression to a specific element
|
|
12
|
+
* - `span` - Optional: character range within text content
|
|
13
|
+
*
|
|
14
|
+
* @category Types
|
|
15
|
+
* @group Observation Types
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```ts
|
|
19
|
+
* // Point to entire snapshot
|
|
20
|
+
* const pointer: SnapshotPointer = {
|
|
21
|
+
* store_id: 'hansard',
|
|
22
|
+
* version: 'AZJx4vM'
|
|
23
|
+
* }
|
|
24
|
+
*
|
|
25
|
+
* // Point to specific speech within a transcript
|
|
26
|
+
* const speechPointer: SnapshotPointer = {
|
|
27
|
+
* store_id: 'hansard',
|
|
28
|
+
* version: 'AZJx4vM',
|
|
29
|
+
* path: '$.content[0].speeches[2]'
|
|
30
|
+
* }
|
|
31
|
+
*
|
|
32
|
+
* // Point to text range within content
|
|
33
|
+
* const rangePointer: SnapshotPointer = {
|
|
34
|
+
* store_id: 'hansard',
|
|
35
|
+
* version: 'AZJx4vM',
|
|
36
|
+
* path: '$.content[0].speeches[2].text',
|
|
37
|
+
* span: { start: 100, end: 250 }
|
|
38
|
+
* }
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
export type SnapshotPointer = {
|
|
42
|
+
store_id: string;
|
|
43
|
+
version: string;
|
|
44
|
+
path?: string;
|
|
45
|
+
span?: {
|
|
46
|
+
start: number;
|
|
47
|
+
end: number;
|
|
48
|
+
};
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Definition for a typed observation schema.
|
|
52
|
+
*
|
|
53
|
+
* Created by `define_observation_type()` and passed to `observations.put()`.
|
|
54
|
+
* The schema validates observation content and provides type inference.
|
|
55
|
+
*
|
|
56
|
+
* @category Types
|
|
57
|
+
* @group Observation Types
|
|
58
|
+
*/
|
|
59
|
+
export type ObservationTypeDef<T> = {
|
|
60
|
+
readonly name: string;
|
|
61
|
+
readonly schema: ZodType<T>;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* A stored observation record linking structured facts to versioned content.
|
|
65
|
+
*
|
|
66
|
+
* Observations are typed facts extracted from or computed about content:
|
|
67
|
+
* - `source` - Points to the content this observation is about
|
|
68
|
+
* - `content` - The typed observation data (validated by schema)
|
|
69
|
+
* - `confidence` - Optional confidence score (0.0 to 1.0)
|
|
70
|
+
* - `observed_at` - When the observation was made (vs when stored)
|
|
71
|
+
* - `derived_from` - Optional provenance chain for computed observations
|
|
72
|
+
*
|
|
73
|
+
* @category Types
|
|
74
|
+
* @group Observation Types
|
|
75
|
+
*
|
|
76
|
+
* @example
|
|
77
|
+
* ```ts
|
|
78
|
+
* const observation: Observation<EntityMention> = {
|
|
79
|
+
* id: 'obs_abc123',
|
|
80
|
+
* type: 'entity_mention',
|
|
81
|
+
* source: { store_id: 'hansard', version: 'AZJx4vM', path: '$.speeches[0]' },
|
|
82
|
+
* content: { entity: 'Climate Change', entity_type: 'topic' },
|
|
83
|
+
* confidence: 0.95,
|
|
84
|
+
* observed_at: new Date('2024-01-15'),
|
|
85
|
+
* created_at: new Date(),
|
|
86
|
+
* derived_from: [{ store_id: 'hansard', version: 'AZJx4vM' }]
|
|
87
|
+
* }
|
|
88
|
+
* ```
|
|
89
|
+
*/
|
|
90
|
+
export type Observation<T = unknown> = {
|
|
91
|
+
id: string;
|
|
92
|
+
type: string;
|
|
93
|
+
source: SnapshotPointer;
|
|
94
|
+
content: T;
|
|
95
|
+
confidence?: number;
|
|
96
|
+
observed_at?: Date;
|
|
97
|
+
created_at: Date;
|
|
98
|
+
derived_from?: SnapshotPointer[];
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* Observation metadata without content payload.
|
|
102
|
+
*
|
|
103
|
+
* Used for efficient listing operations where only metadata is needed.
|
|
104
|
+
*
|
|
105
|
+
* @category Types
|
|
106
|
+
* @group Observation Types
|
|
107
|
+
*/
|
|
108
|
+
export type ObservationMeta = Omit<Observation<never>, 'content'>;
|
|
109
|
+
/**
|
|
110
|
+
* Query options for filtering observations.
|
|
111
|
+
*
|
|
112
|
+
* @category Types
|
|
113
|
+
* @group Observation Types
|
|
114
|
+
*
|
|
115
|
+
* @example
|
|
116
|
+
* ```ts
|
|
117
|
+
* // Find recent entity mentions from a specific store
|
|
118
|
+
* const opts: ObservationQueryOpts = {
|
|
119
|
+
* type: 'entity_mention',
|
|
120
|
+
* source_store: 'hansard',
|
|
121
|
+
* after: new Date('2024-01-01'),
|
|
122
|
+
* limit: 100
|
|
123
|
+
* }
|
|
124
|
+
*
|
|
125
|
+
* // Find all observations for a specific version
|
|
126
|
+
* const versionOpts: ObservationQueryOpts = {
|
|
127
|
+
* source_store: 'hansard',
|
|
128
|
+
* source_version: 'AZJx4vM',
|
|
129
|
+
* include_stale: true
|
|
130
|
+
* }
|
|
131
|
+
* ```
|
|
132
|
+
*/
|
|
133
|
+
export type ObservationQueryOpts = {
|
|
134
|
+
type?: string | string[];
|
|
135
|
+
source_store?: string;
|
|
136
|
+
source_version?: string;
|
|
137
|
+
source_prefix?: string;
|
|
138
|
+
after?: Date;
|
|
139
|
+
before?: Date;
|
|
140
|
+
created_after?: Date;
|
|
141
|
+
created_before?: Date;
|
|
142
|
+
include_stale?: boolean;
|
|
143
|
+
limit?: number;
|
|
144
|
+
cursor?: string;
|
|
145
|
+
};
|
|
146
|
+
/**
|
|
147
|
+
* Options for creating a new observation.
|
|
148
|
+
*
|
|
149
|
+
* @category Types
|
|
150
|
+
* @group Observation Types
|
|
151
|
+
*
|
|
152
|
+
* @example
|
|
153
|
+
* ```ts
|
|
154
|
+
* const opts: ObservationPutOpts<EntityMention> = {
|
|
155
|
+
* source: { store_id: 'hansard', version: 'AZJx4vM', path: '$.speeches[0]' },
|
|
156
|
+
* content: { entity: 'Climate Change', entity_type: 'topic' },
|
|
157
|
+
* confidence: 0.95,
|
|
158
|
+
* observed_at: new Date(),
|
|
159
|
+
* derived_from: [{ store_id: 'raw_text', version: 'xyz789' }]
|
|
160
|
+
* }
|
|
161
|
+
* ```
|
|
162
|
+
*/
|
|
163
|
+
export type ObservationPutOpts<T> = {
|
|
164
|
+
source: SnapshotPointer;
|
|
165
|
+
content: T;
|
|
166
|
+
confidence?: number;
|
|
167
|
+
observed_at?: Date;
|
|
168
|
+
derived_from?: SnapshotPointer[];
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Defines a typed observation schema.
|
|
172
|
+
*
|
|
173
|
+
* Creates an ObservationTypeDef that provides:
|
|
174
|
+
* - Runtime validation via Zod schema
|
|
175
|
+
* - Compile-time type inference for content
|
|
176
|
+
* - Named type for querying and filtering
|
|
177
|
+
*
|
|
178
|
+
* @category Core
|
|
179
|
+
* @group Observation Helpers
|
|
180
|
+
* @param name - Unique identifier for this observation type
|
|
181
|
+
* @param schema - Zod schema for validating observation content
|
|
182
|
+
* @returns An ObservationTypeDef to pass to `observations.put()`
|
|
183
|
+
*
|
|
184
|
+
* @example
|
|
185
|
+
* ```ts
|
|
186
|
+
* import { z } from 'zod'
|
|
187
|
+
*
|
|
188
|
+
* const EntityMentionSchema = z.object({
|
|
189
|
+
* entity: z.string(),
|
|
190
|
+
* entity_type: z.enum(['person', 'organization', 'topic', 'location']),
|
|
191
|
+
* context: z.string().optional()
|
|
192
|
+
* })
|
|
193
|
+
*
|
|
194
|
+
* const entity_mention = define_observation_type('entity_mention', EntityMentionSchema)
|
|
195
|
+
*
|
|
196
|
+
* // Type-safe usage
|
|
197
|
+
* await corpus.observations.put(entity_mention, {
|
|
198
|
+
* source: { store_id: 'hansard', version: 'abc123' },
|
|
199
|
+
* content: { entity: 'Parliament', entity_type: 'organization' }
|
|
200
|
+
* })
|
|
201
|
+
* ```
|
|
202
|
+
*/
|
|
203
|
+
export declare function define_observation_type<T>(name: string, schema: ZodType<T>): ObservationTypeDef<T>;
|
|
204
|
+
/**
|
|
205
|
+
* Utility type to extract the content type from an ObservationTypeDef.
|
|
206
|
+
*
|
|
207
|
+
* @category Types
|
|
208
|
+
* @group Observation Types
|
|
209
|
+
*
|
|
210
|
+
* @example
|
|
211
|
+
* ```ts
|
|
212
|
+
* const entity_mention = define_observation_type('entity_mention', EntityMentionSchema)
|
|
213
|
+
*
|
|
214
|
+
* type EntityMention = InferObservationContent<typeof entity_mention>
|
|
215
|
+
* // => { entity: string; entity_type: 'person' | 'organization' | ... }
|
|
216
|
+
* ```
|
|
217
|
+
*/
|
|
218
|
+
export type InferObservationContent<T> = T extends ObservationTypeDef<infer C> ? C : never;
|
|
219
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../observations/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,KAAK,CAAA;AAElC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE;QACL,KAAK,EAAE,MAAM,CAAA;QACb,GAAG,EAAE,MAAM,CAAA;KACZ,CAAA;CACF,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,kBAAkB,CAAC,CAAC,IAAI;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;CAC5B,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI;IACrC,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,eAAe,CAAA;IACvB,OAAO,EAAE,CAAC,CAAA;IACV,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,IAAI,CAAA;IAClB,UAAU,EAAE,IAAI,CAAA;IAChB,YAAY,CAAC,EAAE,eAAe,EAAE,CAAA;CACjC,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAA;AAEjE;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;IACxB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,KAAK,CAAC,EAAE,IAAI,CAAA;IACZ,MAAM,CAAC,EAAE,IAAI,CAAA;IACb,aAAa,CAAC,EAAE,IAAI,CAAA;IACpB,cAAc,CAAC,EAAE,IAAI,CAAA;IACrB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,CAAA;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,kBAAkB,CAAC,CAAC,IAAI;IAClC,MAAM,EAAE,eAAe,CAAA;IACvB,OAAO,EAAE,CAAC,CAAA;IACV,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,IAAI,CAAA;IAClB,YAAY,CAAC,EAAE,eAAe,EAAE,CAAA;CACjC,CAAA;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,uBAAuB,CAAC,CAAC,EACvC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GACjB,kBAAkB,CAAC,CAAC,CAAC,CAEvB;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,MAAM,uBAAuB,CAAC,CAAC,IAAI,CAAC,SAAS,kBAAkB,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module ObservationTypes
|
|
3
|
+
* @description Type definitions for the observations feature.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Defines a typed observation schema.
|
|
7
|
+
*
|
|
8
|
+
* Creates an ObservationTypeDef that provides:
|
|
9
|
+
* - Runtime validation via Zod schema
|
|
10
|
+
* - Compile-time type inference for content
|
|
11
|
+
* - Named type for querying and filtering
|
|
12
|
+
*
|
|
13
|
+
* @category Core
|
|
14
|
+
* @group Observation Helpers
|
|
15
|
+
* @param name - Unique identifier for this observation type
|
|
16
|
+
* @param schema - Zod schema for validating observation content
|
|
17
|
+
* @returns An ObservationTypeDef to pass to `observations.put()`
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```ts
|
|
21
|
+
* import { z } from 'zod'
|
|
22
|
+
*
|
|
23
|
+
* const EntityMentionSchema = z.object({
|
|
24
|
+
* entity: z.string(),
|
|
25
|
+
* entity_type: z.enum(['person', 'organization', 'topic', 'location']),
|
|
26
|
+
* context: z.string().optional()
|
|
27
|
+
* })
|
|
28
|
+
*
|
|
29
|
+
* const entity_mention = define_observation_type('entity_mention', EntityMentionSchema)
|
|
30
|
+
*
|
|
31
|
+
* // Type-safe usage
|
|
32
|
+
* await corpus.observations.put(entity_mention, {
|
|
33
|
+
* source: { store_id: 'hansard', version: 'abc123' },
|
|
34
|
+
* content: { entity: 'Parliament', entity_type: 'organization' }
|
|
35
|
+
* })
|
|
36
|
+
* ```
|
|
37
|
+
*/
|
|
38
|
+
export function define_observation_type(name, schema) {
|
|
39
|
+
return { name, schema };
|
|
40
|
+
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module ObservationUtils
|
|
3
|
+
* @description Utility functions for working with SnapshotPointers and observations.
|
|
4
|
+
*/
|
|
5
|
+
import type { SnapshotPointer } from './types';
|
|
6
|
+
import type { Result, CorpusError } from '../types';
|
|
7
|
+
/**
|
|
8
|
+
* Creates a SnapshotPointer to a location in a snapshot.
|
|
9
|
+
*
|
|
10
|
+
* @category Utilities
|
|
11
|
+
* @group Pointer Utilities
|
|
12
|
+
* @param store_id - The store identifier
|
|
13
|
+
* @param version - The snapshot version
|
|
14
|
+
* @param path - Optional JSONPath expression to a specific element
|
|
15
|
+
* @param span - Optional character range within text content
|
|
16
|
+
* @returns A SnapshotPointer instance
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```ts
|
|
20
|
+
* // Point to entire snapshot
|
|
21
|
+
* const pointer = create_pointer('hansard', 'AZJx4vM')
|
|
22
|
+
*
|
|
23
|
+
* // Point to specific element
|
|
24
|
+
* const element = create_pointer('hansard', 'AZJx4vM', '$.speeches[0]')
|
|
25
|
+
*
|
|
26
|
+
* // Point to text range
|
|
27
|
+
* const range = create_pointer('hansard', 'AZJx4vM', '$.speeches[0].text', { start: 100, end: 250 })
|
|
28
|
+
* ```
|
|
29
|
+
*/
|
|
30
|
+
export declare function create_pointer(store_id: string, version: string, path?: string, span?: {
|
|
31
|
+
start: number;
|
|
32
|
+
end: number;
|
|
33
|
+
}): SnapshotPointer;
|
|
34
|
+
/**
|
|
35
|
+
* Converts a SnapshotPointer to a stable string key for Map storage.
|
|
36
|
+
*
|
|
37
|
+
* Format: `store_id:version[:path][:start-end]`
|
|
38
|
+
*
|
|
39
|
+
* @category Utilities
|
|
40
|
+
* @group Pointer Utilities
|
|
41
|
+
* @param pointer - The pointer to convert
|
|
42
|
+
* @returns A string key suitable for use as a Map key
|
|
43
|
+
*
|
|
44
|
+
* @example
|
|
45
|
+
* ```ts
|
|
46
|
+
* const key = pointer_to_key({ store_id: 'hansard', version: 'abc' })
|
|
47
|
+
* // => 'hansard:abc'
|
|
48
|
+
*
|
|
49
|
+
* const pathKey = pointer_to_key({ store_id: 'hansard', version: 'abc', path: '$.foo' })
|
|
50
|
+
* // => 'hansard:abc:$.foo'
|
|
51
|
+
*
|
|
52
|
+
* const spanKey = pointer_to_key({ store_id: 'hansard', version: 'abc', span: { start: 0, end: 10 } })
|
|
53
|
+
* // => 'hansard:abc:0-10'
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
export declare function pointer_to_key(pointer: SnapshotPointer): string;
|
|
57
|
+
/**
|
|
58
|
+
* Parses a pointer key back to a SnapshotPointer.
|
|
59
|
+
*
|
|
60
|
+
* Note: This is a best-effort parse - complex paths containing colons may not round-trip perfectly.
|
|
61
|
+
*
|
|
62
|
+
* @category Utilities
|
|
63
|
+
* @group Pointer Utilities
|
|
64
|
+
* @param key - The string key to parse
|
|
65
|
+
* @returns A SnapshotPointer or null if parsing fails
|
|
66
|
+
*
|
|
67
|
+
* @example
|
|
68
|
+
* ```ts
|
|
69
|
+
* const pointer = key_to_pointer('hansard:abc:$.foo:0-10')
|
|
70
|
+
* // => { store_id: 'hansard', version: 'abc', path: '$.foo', span: { start: 0, end: 10 } }
|
|
71
|
+
*
|
|
72
|
+
* const simple = key_to_pointer('hansard:abc')
|
|
73
|
+
* // => { store_id: 'hansard', version: 'abc' }
|
|
74
|
+
*
|
|
75
|
+
* const invalid = key_to_pointer('invalid')
|
|
76
|
+
* // => null
|
|
77
|
+
* ```
|
|
78
|
+
*/
|
|
79
|
+
export declare function key_to_pointer(key: string): SnapshotPointer | null;
|
|
80
|
+
/**
|
|
81
|
+
* Resolves a JSONPath expression against a value.
|
|
82
|
+
*
|
|
83
|
+
* Supports simple dot notation with array indices:
|
|
84
|
+
* - `$` or empty string - Returns the root value
|
|
85
|
+
* - `$.foo` - Property access
|
|
86
|
+
* - `$.foo.bar` - Nested property access
|
|
87
|
+
* - `$.foo[0]` - Array index access
|
|
88
|
+
* - `$.foo[0].bar` - Combined access
|
|
89
|
+
*
|
|
90
|
+
* @category Utilities
|
|
91
|
+
* @group Pointer Utilities
|
|
92
|
+
* @param value - The value to resolve against
|
|
93
|
+
* @param path - The JSONPath expression
|
|
94
|
+
* @returns Result containing the resolved value or an error
|
|
95
|
+
*
|
|
96
|
+
* @example
|
|
97
|
+
* ```ts
|
|
98
|
+
* const data = { speeches: [{ text: 'Hello', speaker: 'Alice' }] }
|
|
99
|
+
*
|
|
100
|
+
* const result = resolve_path(data, '$.speeches[0].text')
|
|
101
|
+
* if (result.ok) console.log(result.value) // 'Hello'
|
|
102
|
+
*
|
|
103
|
+
* const root = resolve_path(data, '$')
|
|
104
|
+
* if (root.ok) console.log(root.value) // { speeches: [...] }
|
|
105
|
+
* ```
|
|
106
|
+
*/
|
|
107
|
+
export declare function resolve_path<T = unknown>(value: unknown, path: string): Result<T, CorpusError>;
|
|
108
|
+
/**
|
|
109
|
+
* Applies a span (character range) to a string value.
|
|
110
|
+
*
|
|
111
|
+
* @category Utilities
|
|
112
|
+
* @group Pointer Utilities
|
|
113
|
+
* @param value - The string to slice
|
|
114
|
+
* @param span - The character range to extract
|
|
115
|
+
* @returns Result containing the substring or an error for invalid spans
|
|
116
|
+
*
|
|
117
|
+
* @example
|
|
118
|
+
* ```ts
|
|
119
|
+
* const result = apply_span('Hello, world!', { start: 0, end: 5 })
|
|
120
|
+
* if (result.ok) console.log(result.value) // 'Hello'
|
|
121
|
+
*
|
|
122
|
+
* const invalid = apply_span('Hi', { start: 0, end: 10 })
|
|
123
|
+
* if (!invalid.ok) console.log(invalid.error.kind) // 'validation_error'
|
|
124
|
+
* ```
|
|
125
|
+
*/
|
|
126
|
+
export declare function apply_span(value: string, span: {
|
|
127
|
+
start: number;
|
|
128
|
+
end: number;
|
|
129
|
+
}): Result<string, CorpusError>;
|
|
130
|
+
/**
|
|
131
|
+
* Generates a unique observation ID.
|
|
132
|
+
*
|
|
133
|
+
* Format: `obs_{timestamp}_{random}` where timestamp is base36-encoded
|
|
134
|
+
* and random is 8 characters of base36.
|
|
135
|
+
*
|
|
136
|
+
* @category Utilities
|
|
137
|
+
* @group Observation Utilities
|
|
138
|
+
* @returns A unique observation ID string
|
|
139
|
+
*
|
|
140
|
+
* @example
|
|
141
|
+
* ```ts
|
|
142
|
+
* const id = generate_observation_id()
|
|
143
|
+
* // => 'obs_lq9x2k_a7b3c2d1'
|
|
144
|
+
* ```
|
|
145
|
+
*/
|
|
146
|
+
export declare function generate_observation_id(): string;
|
|
147
|
+
/**
|
|
148
|
+
* Checks if two SnapshotPointers reference the same location.
|
|
149
|
+
*
|
|
150
|
+
* @category Utilities
|
|
151
|
+
* @group Pointer Utilities
|
|
152
|
+
* @param a - First pointer
|
|
153
|
+
* @param b - Second pointer
|
|
154
|
+
* @returns True if pointers reference the same location
|
|
155
|
+
*
|
|
156
|
+
* @example
|
|
157
|
+
* ```ts
|
|
158
|
+
* const p1 = create_pointer('hansard', 'abc', '$.foo')
|
|
159
|
+
* const p2 = create_pointer('hansard', 'abc', '$.foo')
|
|
160
|
+
* const p3 = create_pointer('hansard', 'xyz', '$.foo')
|
|
161
|
+
*
|
|
162
|
+
* pointers_equal(p1, p2) // true
|
|
163
|
+
* pointers_equal(p1, p3) // false
|
|
164
|
+
* ```
|
|
165
|
+
*/
|
|
166
|
+
export declare function pointers_equal(a: SnapshotPointer, b: SnapshotPointer): boolean;
|
|
167
|
+
/**
|
|
168
|
+
* Creates a pointer to the same snapshot without path or span.
|
|
169
|
+
*
|
|
170
|
+
* @category Utilities
|
|
171
|
+
* @group Pointer Utilities
|
|
172
|
+
* @param pointer - The source pointer
|
|
173
|
+
* @returns A pointer to just the snapshot (store_id + version)
|
|
174
|
+
*
|
|
175
|
+
* @example
|
|
176
|
+
* ```ts
|
|
177
|
+
* const detailed = create_pointer('hansard', 'abc', '$.speeches[0]', { start: 0, end: 100 })
|
|
178
|
+
* const snapshot = pointer_to_snapshot(detailed)
|
|
179
|
+
* // => { store_id: 'hansard', version: 'abc' }
|
|
180
|
+
* ```
|
|
181
|
+
*/
|
|
182
|
+
export declare function pointer_to_snapshot(pointer: SnapshotPointer): SnapshotPointer;
|
|
183
|
+
//# sourceMappingURL=utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../observations/utils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAC9C,OAAO,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AAGnD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GACpC,eAAe,CAKjB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAK/D;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAwBlE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,YAAY,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAsC9F;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAS3G;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,IAAI,MAAM,CAIhD;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,eAAe,GAAG,OAAO,CAO9E;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,eAAe,GAAG,eAAe,CAE7E"}
|