@mindstudio-ai/agent 0.1.176 → 0.1.177
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/analytics/index.d.ts +384 -0
- package/dist/auth/index.d.ts +113 -0
- package/dist/browser/index.d.ts +332 -0
- package/dist/cli.js +27 -27
- package/dist/client.d.ts +767 -0
- package/dist/config.d.ts +13 -0
- package/dist/context.d.ts +125 -0
- package/dist/datasources/index.d.ts +52 -0
- package/dist/datasources/source.d.ts +1074 -0
- package/dist/db/aggregate.d.ts +101 -0
- package/dist/db/index.d.ts +280 -0
- package/dist/db/mutation.d.ts +65 -0
- package/dist/db/predicate.d.ts +85 -0
- package/dist/db/query.d.ts +233 -0
- package/dist/db/raw.d.ts +36 -0
- package/dist/db/readBatching.d.ts +36 -0
- package/dist/db/scanHints.d.ts +8 -0
- package/dist/db/sql.d.ts +136 -0
- package/dist/db/table.d.ts +146 -0
- package/dist/db/time.d.ts +17 -0
- package/dist/db/types.d.ts +237 -0
- package/dist/db-ws.d.ts +35 -0
- package/dist/email/index.d.ts +396 -0
- package/dist/errors.d.ts +34 -0
- package/dist/events/index.d.ts +127 -0
- package/dist/files/index.d.ts +62 -0
- package/dist/files/store.d.ts +244 -0
- package/dist/generated/metadata.d.ts +8 -0
- package/dist/generated/snippets.d.ts +8 -0
- package/dist/generated/steps.d.ts +2745 -0
- package/dist/generated/types.d.ts +4234 -0
- package/dist/http.d.ts +22 -0
- package/dist/index.d.ts +59 -13329
- package/dist/index.js +15 -15
- package/dist/index.js.map +7 -1
- package/dist/jewel/index.d.ts +325 -0
- package/dist/mapper/index.d.ts +229 -0
- package/dist/rate-limit.d.ts +25 -0
- package/dist/reader/index.d.ts +170 -0
- package/dist/respond.d.ts +68 -0
- package/dist/task/index.d.ts +18 -0
- package/dist/task/local.d.ts +48 -0
- package/dist/task/schema.d.ts +120 -0
- package/dist/task/types.d.ts +203 -0
- package/dist/types.d.ts +591 -0
- package/dist/voice/index.d.ts +87 -0
- package/package.json +6 -12
- package/dist/postinstall.js +0 -12115
|
@@ -0,0 +1,1074 @@
|
|
|
1
|
+
/** @internal Transport: `POST /_internal/v2/datasources/<op>` with the hook token. */
|
|
2
|
+
export type DataSourcesTransport = (op: string, body: unknown) => Promise<any>;
|
|
3
|
+
/** Where a retrieved chunk came from — enough to show the user the source. */
|
|
4
|
+
export interface Citation {
|
|
5
|
+
documentId: string;
|
|
6
|
+
filename: string | null;
|
|
7
|
+
/** 1-based. Null for formats with no pagination (plain text, html). */
|
|
8
|
+
pageNumber: number | null;
|
|
9
|
+
/**
|
|
10
|
+
* Position within the document, 0-based.
|
|
11
|
+
*
|
|
12
|
+
* `(documentId, chunkIndex)` is the stable identity of a chunk — use it as
|
|
13
|
+
* the key for an eval set or a regression check rather than matching on
|
|
14
|
+
* `text`. Stable for as long as the corpus keeps its current configuration:
|
|
15
|
+
* re-chunking a source (a different chunk size, say) moves the boundaries and
|
|
16
|
+
* therefore renumbers, which is inherent rather than a wobble.
|
|
17
|
+
*/
|
|
18
|
+
chunkIndex: number | null;
|
|
19
|
+
/** Enclosing headings, outermost first. */
|
|
20
|
+
headingPath: string[];
|
|
21
|
+
/**
|
|
22
|
+
* Region of the page this chunk came from, when the format had a layout to
|
|
23
|
+
* measure — PDFs do, docx and plain text don't. Precise enough to highlight.
|
|
24
|
+
*/
|
|
25
|
+
boundingBox?: {
|
|
26
|
+
topLeftX: number;
|
|
27
|
+
topLeftY: number;
|
|
28
|
+
bottomRightX: number;
|
|
29
|
+
bottomRightY: number;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Stable path on the app's own domain for the source document
|
|
33
|
+
* (`/_/datasources/<sourceId>/documents/<documentId>`), the same shape as a
|
|
34
|
+
* private file's `url`: a signed-in user's same-origin request authorizes
|
|
35
|
+
* against their session and redirects to the file, so there's nothing to
|
|
36
|
+
* sign or await. It is not a public link. For a public app or another
|
|
37
|
+
* origin, mint one that needs no session the way you would for a file:
|
|
38
|
+
* `search(q, { shareCitations: 3600 })` signs every citation in the response,
|
|
39
|
+
* or {@link DataSource.shareUrl} signs one document on demand.
|
|
40
|
+
*/
|
|
41
|
+
url: string;
|
|
42
|
+
}
|
|
43
|
+
/** A metadata value as stored on a document: scalars only. */
|
|
44
|
+
export type MetadataValue = string | number | boolean;
|
|
45
|
+
/**
|
|
46
|
+
* Numeric range over a metadata value (inclusive both ends). Numbers only —
|
|
47
|
+
* to range on dates, store them as sortable integers at add time (epoch
|
|
48
|
+
* seconds or `20240315`-style YYYYMMDD) and range on those.
|
|
49
|
+
*/
|
|
50
|
+
export interface MetadataRange {
|
|
51
|
+
gte?: number;
|
|
52
|
+
lte?: number;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Tags attached to a document when it's added — department, year, doc type,
|
|
56
|
+
* a per-user scope — and matched by {@link SearchFilter.metadata} at query
|
|
57
|
+
* time. Up to 16 keys per document; keys are alphanumeric with `_`/`-`.
|
|
58
|
+
*/
|
|
59
|
+
export type DocumentMetadata = Record<string, MetadataValue>;
|
|
60
|
+
/**
|
|
61
|
+
* Which retrieval branches run for a search.
|
|
62
|
+
*
|
|
63
|
+
* - `hybrid` — semantic and keyword retrieval fused. The default.
|
|
64
|
+
* - `semantic` — the embedding alone (what `hybrid: false` selects).
|
|
65
|
+
* - `lexical` — keyword matching alone, with NO query embedding. The cheapest
|
|
66
|
+
* and fastest mode; right when the query is an identifier (an error code, a
|
|
67
|
+
* SKU, a name) rather than a meaning.
|
|
68
|
+
*/
|
|
69
|
+
export type SearchMode = 'hybrid' | 'semantic' | 'lexical';
|
|
70
|
+
/**
|
|
71
|
+
* Narrow a search before ranking. Every condition ANDs with the others, and a
|
|
72
|
+
* filter can only ever narrow — it runs inside your corpus, not across it.
|
|
73
|
+
*/
|
|
74
|
+
export interface SearchFilter {
|
|
75
|
+
/**
|
|
76
|
+
* Match document metadata set at add time. A scalar must equal; an array
|
|
77
|
+
* matches any of its values; `{ gte?, lte? }` matches a numeric range. Keys
|
|
78
|
+
* AND together:
|
|
79
|
+
* `{ department: 'legal', year: [2025, 2026], publishedAt: { gte: 20250101 } }`.
|
|
80
|
+
*/
|
|
81
|
+
metadata?: Record<string, MetadataValue | MetadataValue[] | MetadataRange>;
|
|
82
|
+
/** Exact filename, or any of several. */
|
|
83
|
+
filename?: string | string[];
|
|
84
|
+
/** Restrict to specific documents (ids from {@link DataSource.documents}). */
|
|
85
|
+
documentIds?: string[];
|
|
86
|
+
/**
|
|
87
|
+
* Passages about any of these entities: the ids their documents assert
|
|
88
|
+
* ({@link AddOptions.entities}) or their text mentions by a name in the
|
|
89
|
+
* source's dictionary ({@link DataSource.pushDictionary}). Up to 256; the
|
|
90
|
+
* usual shape is a neighbourhood your own tables walked out to, then
|
|
91
|
+
* searched.
|
|
92
|
+
*/
|
|
93
|
+
entities?: string[];
|
|
94
|
+
/** Page range, inclusive — for paged formats like PDF. */
|
|
95
|
+
pages?: {
|
|
96
|
+
min?: number;
|
|
97
|
+
max?: number;
|
|
98
|
+
};
|
|
99
|
+
/** Chunk text must contain ALL of these words, in any order. */
|
|
100
|
+
contains?: string;
|
|
101
|
+
/** Chunk text must contain this exact word sequence, adjacent and in order. */
|
|
102
|
+
phrase?: string;
|
|
103
|
+
}
|
|
104
|
+
/** Where one retrieval branch put a hit, and what that branch scored it. */
|
|
105
|
+
export interface BranchPosition {
|
|
106
|
+
/** 0-based position within that branch's own results. */
|
|
107
|
+
rank: number;
|
|
108
|
+
score: number;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Which half of hybrid retrieval found a hit. Only present when `explain` was
|
|
112
|
+
* requested — see {@link SearchOptions.explain}.
|
|
113
|
+
*/
|
|
114
|
+
export interface SearchExplain {
|
|
115
|
+
/** Semantic (embedding) retrieval. Null if this branch didn't find the hit. */
|
|
116
|
+
dense: BranchPosition | null;
|
|
117
|
+
/** Keyword/IDF retrieval. Null when `hybrid` is off, or if it didn't find it. */
|
|
118
|
+
lexical: BranchPosition | null;
|
|
119
|
+
matchedVia: 'dense' | 'lexical' | 'both';
|
|
120
|
+
}
|
|
121
|
+
/** One passage of a listing ({@link DataSource.passages}): a search hit without the ranking. */
|
|
122
|
+
export interface DataSourcePassage {
|
|
123
|
+
/** The chunk, prefixed with its heading path for context. */
|
|
124
|
+
text: string;
|
|
125
|
+
/** The ids the passage carries: its document's, and those its text names from the dictionary. */
|
|
126
|
+
entities: string[];
|
|
127
|
+
citation: Citation;
|
|
128
|
+
}
|
|
129
|
+
/** A page of {@link DataSource.passages}. */
|
|
130
|
+
export interface DataSourcePassagePage {
|
|
131
|
+
passages: DataSourcePassage[];
|
|
132
|
+
/** Pass back for the next page; null after the last. */
|
|
133
|
+
nextCursor: string | null;
|
|
134
|
+
latencyMs: number;
|
|
135
|
+
}
|
|
136
|
+
export interface SearchHit {
|
|
137
|
+
/**
|
|
138
|
+
* Relevance of this hit. Comparable within a response, not across them —
|
|
139
|
+
* the scale depends on how the search ran (check {@link SearchRan}):
|
|
140
|
+
* cosine similarity in `semantic` mode, an RRF rank reciprocal in `hybrid`
|
|
141
|
+
* (small numbers that are not similarities), keyword-overlap weight in
|
|
142
|
+
* `lexical`. When reranking ran, it's the reranker's 0–1 relevance instead,
|
|
143
|
+
* with the retrieval value preserved as `retrievalScore` — so rerank-on
|
|
144
|
+
* `score` is the one scale that's stable across modes, and the right place
|
|
145
|
+
* for quality cutoffs.
|
|
146
|
+
*/
|
|
147
|
+
score: number;
|
|
148
|
+
/** The matched chunk, prefixed with its heading path for context. */
|
|
149
|
+
text: string;
|
|
150
|
+
/**
|
|
151
|
+
* The entities this passage carries (its document's asserted ids, and the
|
|
152
|
+
* ids its text names from the source's dictionary), so you can widen from
|
|
153
|
+
* what you found: walk your own tables out from them, then search again
|
|
154
|
+
* with {@link SearchFilter.entities}. Empty when none.
|
|
155
|
+
*/
|
|
156
|
+
entities: string[];
|
|
157
|
+
citation: Citation;
|
|
158
|
+
/**
|
|
159
|
+
* Where retrieval put this hit BEFORE reranking, and what it scored.
|
|
160
|
+
*
|
|
161
|
+
* With reranking on, `score` is the reranker's relevance score and this is
|
|
162
|
+
* the retriever's — different quantities, so they're kept apart rather than
|
|
163
|
+
* blended. Comparing `retrievalRank` with the hit's final position is how you
|
|
164
|
+
* see what reranking actually did ("retrieved 7th, reranked to 1st").
|
|
165
|
+
*
|
|
166
|
+
* Named for the stage rather than the method: it's a fused hybrid score when
|
|
167
|
+
* `hybrid` is on and a cosine similarity when it's off.
|
|
168
|
+
*/
|
|
169
|
+
retrievalRank?: number;
|
|
170
|
+
retrievalScore?: number;
|
|
171
|
+
/** Only when `explain` was requested. */
|
|
172
|
+
explain?: SearchExplain;
|
|
173
|
+
/** Only when `expand` was requested. Outermost first, so `[...before, text, ...after]` reads in order. */
|
|
174
|
+
neighbors?: {
|
|
175
|
+
before: string[];
|
|
176
|
+
after: string[];
|
|
177
|
+
};
|
|
178
|
+
/**
|
|
179
|
+
* Only when `highlight` was requested: where the query's most distinctive
|
|
180
|
+
* terms land in `text`, as `text.slice(start, end)` ranges, each carrying the
|
|
181
|
+
* `token` it matched so you can colour or group by term.
|
|
182
|
+
*
|
|
183
|
+
* NOT every query term — see {@link SearchOptions.highlight}. Keyword-based,
|
|
184
|
+
* so a hit that matched semantically may report an empty array, which is
|
|
185
|
+
* itself informative.
|
|
186
|
+
*/
|
|
187
|
+
matches?: {
|
|
188
|
+
start: number;
|
|
189
|
+
end: number;
|
|
190
|
+
token: string;
|
|
191
|
+
}[];
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* What a search actually ran — as opposed to what was asked for.
|
|
195
|
+
*
|
|
196
|
+
* Worth checking when a result surprises you: an omitted `mode` falls back to
|
|
197
|
+
* the corpus's own configuration, and `reranked` is false when reranking was
|
|
198
|
+
* skipped or failed open. Absent only when the data source doesn't exist yet,
|
|
199
|
+
* in which case nothing ran at all.
|
|
200
|
+
*/
|
|
201
|
+
export interface SearchRan {
|
|
202
|
+
search: SearchMode;
|
|
203
|
+
hybrid: boolean;
|
|
204
|
+
reranked: boolean;
|
|
205
|
+
/** Which build served the query. Changes when a re-vectorization is promoted. */
|
|
206
|
+
pipelineVersion: number;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Per-query overrides.
|
|
210
|
+
*
|
|
211
|
+
* These are the settings that are free to change: none of them touch a stored
|
|
212
|
+
* vector, so they take effect on the next call and cost nothing. Anything that
|
|
213
|
+
* would require rebuilding the corpus — chunking, the embedding model, whether
|
|
214
|
+
* chunks are contextualized — is a property of the data source, configured
|
|
215
|
+
* with `remy-admin datasources config` rather than passed here.
|
|
216
|
+
*
|
|
217
|
+
* The defaults come from the data source's own configuration, so most callers
|
|
218
|
+
* should pass nothing.
|
|
219
|
+
*/
|
|
220
|
+
export interface SearchOptions {
|
|
221
|
+
/** Results to return. Default 5, capped at 50. */
|
|
222
|
+
topK?: number;
|
|
223
|
+
/**
|
|
224
|
+
* Floor on the RETRIEVAL score — applied to the retrieval branch, never to
|
|
225
|
+
* the fused hybrid score or the reranker's score. In `semantic` and
|
|
226
|
+
* `hybrid` modes it's a cosine floor on the embedding branch; in `lexical`,
|
|
227
|
+
* a keyword-overlap floor. The scale depends on the embedding model, so
|
|
228
|
+
* measure before using — and with reranking on, prefer cutting on each
|
|
229
|
+
* hit's returned `score` (the reranker's 0–1 relevance) instead.
|
|
230
|
+
*/
|
|
231
|
+
scoreThreshold?: number;
|
|
232
|
+
/**
|
|
233
|
+
* Narrow the search to matching chunks before ranking — by document
|
|
234
|
+
* metadata, filename, document ids, page range, or required words/phrases.
|
|
235
|
+
* See {@link SearchFilter}.
|
|
236
|
+
*/
|
|
237
|
+
filter?: SearchFilter;
|
|
238
|
+
/**
|
|
239
|
+
* Which retrieval branches run. Defaults to the source's configuration
|
|
240
|
+
* (hybrid). `'lexical'` skips the query embedding entirely — fastest, and
|
|
241
|
+
* right for identifier-shaped queries. See {@link SearchMode}.
|
|
242
|
+
*/
|
|
243
|
+
mode?: SearchMode;
|
|
244
|
+
/**
|
|
245
|
+
* At most this many hits per document, backfilled from other documents —
|
|
246
|
+
* stops one document from monopolizing the results. Useful whenever the
|
|
247
|
+
* answer should draw on several sources.
|
|
248
|
+
*/
|
|
249
|
+
maxPerDocument?: number;
|
|
250
|
+
/**
|
|
251
|
+
* Return {@link SearchHit.matches} on each hit: where query terms land in its
|
|
252
|
+
* text, for rendering highlights.
|
|
253
|
+
*
|
|
254
|
+
* Only the query's **distinctive** terms are marked. English function words —
|
|
255
|
+
* `the`, `for`, `is`, `of` — are never marked, and when a passage holds more
|
|
256
|
+
* matches than it can usefully show, the rarest terms win the space. So
|
|
257
|
+
* `what is the policy for parental leave` marks `policy`, `parental` and
|
|
258
|
+
* `leave`, and nothing else. Without that a natural-language query lights up
|
|
259
|
+
* most of the passage and you would need your own stopword list to render
|
|
260
|
+
* anything. Words you filtered on (`contains`, `phrase`) are always marked.
|
|
261
|
+
*/
|
|
262
|
+
highlight?: boolean;
|
|
263
|
+
/**
|
|
264
|
+
* Rerank results with a cross-encoder before returning them. On by default.
|
|
265
|
+
*
|
|
266
|
+
* Turn it off on a latency-sensitive path — it adds a round trip for a
|
|
267
|
+
* meaningful ranking improvement, which is usually the right trade but not
|
|
268
|
+
* always.
|
|
269
|
+
*/
|
|
270
|
+
rerank?: boolean;
|
|
271
|
+
/**
|
|
272
|
+
* Combine semantic search with exact keyword matching. On by default.
|
|
273
|
+
*
|
|
274
|
+
* Keyword matching is what finds part numbers, error codes and proper nouns
|
|
275
|
+
* that an embedding model never learned. Rarely worth disabling.
|
|
276
|
+
* `hybrid: false` is the same as `mode: 'semantic'`; prefer `mode`, which
|
|
277
|
+
* also offers `'lexical'`.
|
|
278
|
+
*/
|
|
279
|
+
hybrid?: boolean;
|
|
280
|
+
/**
|
|
281
|
+
* Report which branch found each hit, and where each ranked it.
|
|
282
|
+
*
|
|
283
|
+
* A debugging aid, off by default because it costs two extra round trips: a
|
|
284
|
+
* fused result carries one blended score, so the branches have to be asked
|
|
285
|
+
* separately. Results and their order are identical either way — this only
|
|
286
|
+
* adds {@link SearchHit.explain}.
|
|
287
|
+
*/
|
|
288
|
+
explain?: boolean;
|
|
289
|
+
/**
|
|
290
|
+
* Also return this many chunks either side of each hit, in
|
|
291
|
+
* {@link SearchHit.neighbors} — for showing a passage in context. 0-2.
|
|
292
|
+
*
|
|
293
|
+
* `text` is untouched, so citations and highlighting still point at the
|
|
294
|
+
* chunk that actually matched.
|
|
295
|
+
*/
|
|
296
|
+
expand?: number;
|
|
297
|
+
/**
|
|
298
|
+
* Return every `citation.url` as an absolute, signed link that works with
|
|
299
|
+
* **no** session, valid for this many seconds (`true` = 24h; clamped to
|
|
300
|
+
* 60s–90d). The same link {@link DataSource.shareUrl} mints one at a time,
|
|
301
|
+
* done for the whole response in one round trip — the render path of a
|
|
302
|
+
* public search tool. Each link is a bearer capability until it expires, so
|
|
303
|
+
* sign at render time rather than storing signed results.
|
|
304
|
+
*/
|
|
305
|
+
shareCitations?: number | true;
|
|
306
|
+
}
|
|
307
|
+
export interface AddOptions {
|
|
308
|
+
/**
|
|
309
|
+
* Required — the extension selects the extraction route. PDFs and office
|
|
310
|
+
* formats go to a document model; text and CSV are read directly.
|
|
311
|
+
*/
|
|
312
|
+
filename: string;
|
|
313
|
+
contentType?: string;
|
|
314
|
+
/**
|
|
315
|
+
* Tags to attach — filterable at search time via
|
|
316
|
+
* {@link SearchFilter.metadata}. Scalars only, up to 16 keys. Re-adding the
|
|
317
|
+
* same bytes with different metadata updates the tags in place with no
|
|
318
|
+
* re-processing; supplying metadata replaces the whole object.
|
|
319
|
+
*/
|
|
320
|
+
metadata?: DocumentMetadata;
|
|
321
|
+
/**
|
|
322
|
+
* The things this document is about, as your own ids (a tag number, a row
|
|
323
|
+
* id): up to 256, each 1–128 printable ASCII characters without whitespace
|
|
324
|
+
* or commas. Every passage of the document carries them, so
|
|
325
|
+
* {@link SearchFilter.entities} finds passages about an entity and
|
|
326
|
+
* {@link DataSource.entities} counts them. Metadata is facts about the
|
|
327
|
+
* document to filter by; entities are what its passages are about.
|
|
328
|
+
* Re-adding the same bytes with different entities updates them in place;
|
|
329
|
+
* supplying them replaces the whole set, and `[]` clears it.
|
|
330
|
+
*/
|
|
331
|
+
entities?: string[];
|
|
332
|
+
/**
|
|
333
|
+
* On a mapped source, the budget in dollars of the job that takes an
|
|
334
|
+
* object's documents past the first thousand ({@link AddResult.job}): it
|
|
335
|
+
* pauses there, and a plan projected over it waits for an approval rather
|
|
336
|
+
* than starting. Without one the job runs within its plan's projection,
|
|
337
|
+
* with headroom.
|
|
338
|
+
*/
|
|
339
|
+
budgetDollars?: number;
|
|
340
|
+
}
|
|
341
|
+
/** What {@link DataSource.add} returns. */
|
|
342
|
+
export interface AddResult {
|
|
343
|
+
/**
|
|
344
|
+
* The document, or the first of them on a mapped source. Null only when a
|
|
345
|
+
* mapper answered `deletes` — nothing was added.
|
|
346
|
+
*/
|
|
347
|
+
document: DataSourceDocument | null;
|
|
348
|
+
/**
|
|
349
|
+
* Every document the bytes became, or the first thousand of an object that
|
|
350
|
+
* maps to more ({@link job} takes the rest). One entry on an unmapped
|
|
351
|
+
* source.
|
|
352
|
+
*/
|
|
353
|
+
documents: DataSourceDocument[];
|
|
354
|
+
/** How many documents the bytes map to in all: `documents.length` unless {@link job} is set. */
|
|
355
|
+
documentCount: number;
|
|
356
|
+
/** Whether any of them was queued for a build (the rest were already current). */
|
|
357
|
+
queued: boolean;
|
|
358
|
+
/**
|
|
359
|
+
* How the source took the bytes: `unmapped` (no mapper — the bytes are the
|
|
360
|
+
* document), or the mapper's outcome.
|
|
361
|
+
*/
|
|
362
|
+
outcome: 'unmapped' | 'documents' | 'passthrough' | 'deletes';
|
|
363
|
+
/** Documents removed or superseded as a result — a `deletes` outcome, a `replaces`. */
|
|
364
|
+
removed: number;
|
|
365
|
+
/**
|
|
366
|
+
* The documents the mapper produced that failed validation and were not
|
|
367
|
+
* registered, by their index in its output (`document: 1234` is the
|
|
368
|
+
* 1,235th it returned or yielded), with why; null when every one passed.
|
|
369
|
+
* The rest are registered. `add()` throws `mapper_error` only when none
|
|
370
|
+
* passed.
|
|
371
|
+
*/
|
|
372
|
+
invalid: {
|
|
373
|
+
count: number;
|
|
374
|
+
examples: {
|
|
375
|
+
document: number;
|
|
376
|
+
reason: string;
|
|
377
|
+
}[];
|
|
378
|
+
} | null;
|
|
379
|
+
/**
|
|
380
|
+
* The job registering and building the rest of an object that maps to more
|
|
381
|
+
* than a thousand documents; null otherwise. It plans and starts on its own
|
|
382
|
+
* (within {@link AddOptions.budgetDollars}); follow it with
|
|
383
|
+
* {@link DataSource.job}. What the object supersedes or retires goes when
|
|
384
|
+
* the job finishes, once every one of its documents is searchable.
|
|
385
|
+
*/
|
|
386
|
+
job: DataSourceJob | null;
|
|
387
|
+
}
|
|
388
|
+
export interface DataSourceDocument {
|
|
389
|
+
id: string;
|
|
390
|
+
filename: string | null;
|
|
391
|
+
status: 'processing' | 'done' | 'error';
|
|
392
|
+
errorMessage: string | null;
|
|
393
|
+
chunkCount: number | null;
|
|
394
|
+
pageCount: number | null;
|
|
395
|
+
/** Tags set at add time. See {@link AddOptions.metadata}. */
|
|
396
|
+
metadata: DocumentMetadata | null;
|
|
397
|
+
/** Entities the document asserts, sorted. See {@link AddOptions.entities}. */
|
|
398
|
+
entities: string[];
|
|
399
|
+
createdAt: string;
|
|
400
|
+
ingestedAt: string | null;
|
|
401
|
+
}
|
|
402
|
+
/** One page of a corpus. See {@link DataSource.documentsPage}. */
|
|
403
|
+
export interface DataSourceDocumentPage {
|
|
404
|
+
documents: DataSourceDocument[];
|
|
405
|
+
/** Pass to the next call; null when this was the last page. */
|
|
406
|
+
nextCursor: string | null;
|
|
407
|
+
}
|
|
408
|
+
/** One chunk exactly as it was indexed. See {@link DataSource.chunks}. */
|
|
409
|
+
export interface DataSourceChunk {
|
|
410
|
+
index: number;
|
|
411
|
+
text: string;
|
|
412
|
+
pageNumber: number;
|
|
413
|
+
headingPath: string[];
|
|
414
|
+
/**
|
|
415
|
+
* Offsets into the page's extracted markdown. Null for PDFs, which carry a
|
|
416
|
+
* `boundingBox` instead — there is no character stream to point into.
|
|
417
|
+
*/
|
|
418
|
+
charStart: number | null;
|
|
419
|
+
charEnd: number | null;
|
|
420
|
+
boundingBox?: {
|
|
421
|
+
topLeftX: number;
|
|
422
|
+
topLeftY: number;
|
|
423
|
+
bottomRightX: number;
|
|
424
|
+
bottomRightY: number;
|
|
425
|
+
};
|
|
426
|
+
/** The entities this chunk carries, as search hits report them. */
|
|
427
|
+
entities: string[];
|
|
428
|
+
/** base64 of a Float32Array. Only when `vectors: true` was passed. */
|
|
429
|
+
vector?: string;
|
|
430
|
+
}
|
|
431
|
+
/** How a corpus was built and what is in it. See {@link DataSource.stats}. */
|
|
432
|
+
export interface DataSourceStats {
|
|
433
|
+
/** False for a source nothing has created yet — everything else reads zero. */
|
|
434
|
+
exists: boolean;
|
|
435
|
+
documentCount: number;
|
|
436
|
+
counts: {
|
|
437
|
+
total: number;
|
|
438
|
+
done: number;
|
|
439
|
+
processing: number;
|
|
440
|
+
error: number;
|
|
441
|
+
};
|
|
442
|
+
chunkCount: number;
|
|
443
|
+
/** Original document bytes, not index size. */
|
|
444
|
+
storageBytes: number;
|
|
445
|
+
lastIngestedAt: string | null;
|
|
446
|
+
/**
|
|
447
|
+
* The configuration these documents were actually built with — not the
|
|
448
|
+
* platform default, and not necessarily the newest. Changing it is an
|
|
449
|
+
* explicit, owner-triggered migration.
|
|
450
|
+
*/
|
|
451
|
+
pipeline: {
|
|
452
|
+
version: number;
|
|
453
|
+
embeddingModelId: string;
|
|
454
|
+
dimensions: number;
|
|
455
|
+
chunking: {
|
|
456
|
+
strategy: string;
|
|
457
|
+
version: number;
|
|
458
|
+
maxChars: number;
|
|
459
|
+
minChars: number;
|
|
460
|
+
dropBlockTypes: string[];
|
|
461
|
+
};
|
|
462
|
+
contextual: {
|
|
463
|
+
enabled: boolean;
|
|
464
|
+
modelId: string | null;
|
|
465
|
+
};
|
|
466
|
+
images: {
|
|
467
|
+
describe: boolean;
|
|
468
|
+
modelId: string | null;
|
|
469
|
+
};
|
|
470
|
+
} | null;
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
473
|
+
* Which documents a bulk removal selects. The row-level half of
|
|
474
|
+
* {@link SearchFilter} — `metadata`, `filename`, `documentIds` — plus
|
|
475
|
+
* `externalIdPrefix`, the identity a bulk job or S3 connector recorded for each
|
|
476
|
+
* document (a customer's object key, say). Everything ANDs. Chunk-level fields
|
|
477
|
+
* (`pages`, `contains`, `phrase`) do not select documents and are refused, and
|
|
478
|
+
* so is an empty selector: whole-source deletion is an owner operation
|
|
479
|
+
* (`remy-admin datasources delete`), never something app code does.
|
|
480
|
+
*/
|
|
481
|
+
export interface DocumentSelector {
|
|
482
|
+
metadata?: Record<string, MetadataValue | MetadataValue[] | MetadataRange>;
|
|
483
|
+
filename?: string | string[];
|
|
484
|
+
documentIds?: string[];
|
|
485
|
+
/** Documents whose recorded external id starts with this — e.g. a key prefix. */
|
|
486
|
+
externalIdPrefix?: string;
|
|
487
|
+
}
|
|
488
|
+
/**
|
|
489
|
+
* planning → planned → running ⇄ paused → done | failed | cancelled.
|
|
490
|
+
*
|
|
491
|
+
* A `planned` job is waiting for a human: its plan exceeded the budget or the
|
|
492
|
+
* capacity of where the source lives (`error` says which), and the owner
|
|
493
|
+
* approves it with `remy-admin datasources jobs approve <id>`.
|
|
494
|
+
*/
|
|
495
|
+
export type DataSourceJobState = 'planning' | 'planned' | 'running' | 'paused' | 'done' | 'failed' | 'cancelled';
|
|
496
|
+
/**
|
|
497
|
+
* What a job would cost before any of it is spent — an extrapolation from a
|
|
498
|
+
* sample of about a hundred documents at today's rates. `credits` are in
|
|
499
|
+
* nano-dollars; divide by 1e9 for dollars.
|
|
500
|
+
*/
|
|
501
|
+
export interface DataSourceJobPlan {
|
|
502
|
+
objects: number;
|
|
503
|
+
bytes: number;
|
|
504
|
+
/**
|
|
505
|
+
* Mapped sources only: what the mapper did with the sampled objects. The
|
|
506
|
+
* skip share is the baseline the run's skip gate pauses against.
|
|
507
|
+
*/
|
|
508
|
+
mapping: {
|
|
509
|
+
objects: number;
|
|
510
|
+
documentsPerObject: number;
|
|
511
|
+
passthroughShare: number;
|
|
512
|
+
skipShare: number;
|
|
513
|
+
errorShare: number;
|
|
514
|
+
deleteShare: number;
|
|
515
|
+
secondsPerObject: number;
|
|
516
|
+
/** What the mapper's own model and step calls billed per sampled object. */
|
|
517
|
+
creditsPerObject?: number;
|
|
518
|
+
} | null;
|
|
519
|
+
projected: {
|
|
520
|
+
documents: number;
|
|
521
|
+
chunks: number;
|
|
522
|
+
tokens: number;
|
|
523
|
+
credits: {
|
|
524
|
+
embedding: number;
|
|
525
|
+
extraction: number;
|
|
526
|
+
contextual: number;
|
|
527
|
+
/** Mapped sources: the mapper's own calls over every object, the sample's included. */
|
|
528
|
+
mapping?: number;
|
|
529
|
+
total: number;
|
|
530
|
+
};
|
|
531
|
+
};
|
|
532
|
+
capacity: {
|
|
533
|
+
placement: 'shared' | 'dedicated';
|
|
534
|
+
maxPoints: number;
|
|
535
|
+
heldPoints: number;
|
|
536
|
+
/** False means the job will not run until the source moves or the resource grows. */
|
|
537
|
+
fits: boolean;
|
|
538
|
+
};
|
|
539
|
+
durationMinutes: number;
|
|
540
|
+
warnings: string[];
|
|
541
|
+
computedAt: string;
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* A bulk ingestion job — a sync, a corpus load started from the CLI, a remap
|
|
545
|
+
* of a mapped source's raw copies, a replay of an earlier job's quarantine, a
|
|
546
|
+
* reread of the source's documents by its reader, or the rest of one large
|
|
547
|
+
* object an {@link DataSource.add} took.
|
|
548
|
+
*/
|
|
549
|
+
export interface DataSourceJob {
|
|
550
|
+
id: string;
|
|
551
|
+
state: DataSourceJobState;
|
|
552
|
+
source: {
|
|
553
|
+
type: 'store';
|
|
554
|
+
store: string;
|
|
555
|
+
access: string;
|
|
556
|
+
prefix: string;
|
|
557
|
+
} | {
|
|
558
|
+
type: 'manifest';
|
|
559
|
+
key: string;
|
|
560
|
+
} | {
|
|
561
|
+
type: 'connector';
|
|
562
|
+
connectorId: string;
|
|
563
|
+
} | {
|
|
564
|
+
type: 'sample';
|
|
565
|
+
fromDataSourceId: string;
|
|
566
|
+
size: number;
|
|
567
|
+
} | {
|
|
568
|
+
type: 'remap';
|
|
569
|
+
} | {
|
|
570
|
+
type: 'replay';
|
|
571
|
+
fromJobId: string;
|
|
572
|
+
kind: 'skip' | 'error' | null;
|
|
573
|
+
} | {
|
|
574
|
+
type: 'reread';
|
|
575
|
+
unread: boolean;
|
|
576
|
+
} | {
|
|
577
|
+
type: 'object';
|
|
578
|
+
ref: string;
|
|
579
|
+
};
|
|
580
|
+
/**
|
|
581
|
+
* Objects the mapper skipped or failed on, and documents the reader failed
|
|
582
|
+
* on; null where the job was read from a list.
|
|
583
|
+
*/
|
|
584
|
+
quarantine: {
|
|
585
|
+
skipped: number;
|
|
586
|
+
errors: number;
|
|
587
|
+
readErrors: number;
|
|
588
|
+
} | null;
|
|
589
|
+
plan: DataSourceJobPlan | null;
|
|
590
|
+
/** The ceiling this job pauses at, in dollars; null when none was set. */
|
|
591
|
+
budgetDollars: number | null;
|
|
592
|
+
/**
|
|
593
|
+
* Spend so far: embedding, extraction and contextual at catalog rates, plus
|
|
594
|
+
* what the mapper's own calls billed. An estimate, not the ledger; the
|
|
595
|
+
* budget pauses the job on it.
|
|
596
|
+
*/
|
|
597
|
+
estimatedDollars: number;
|
|
598
|
+
/**
|
|
599
|
+
* The mapper's and reader's own calls' part of `estimatedDollars`, the
|
|
600
|
+
* plan's sample included.
|
|
601
|
+
*/
|
|
602
|
+
mapperDollars?: number;
|
|
603
|
+
/**
|
|
604
|
+
* A running job's read stage on a source with a reader: documents read of
|
|
605
|
+
* those its builds handed the reader. A reread counts its reads in `counts`.
|
|
606
|
+
*/
|
|
607
|
+
reading?: {
|
|
608
|
+
documentsRead: number;
|
|
609
|
+
documentsTotal: number;
|
|
610
|
+
} | null;
|
|
611
|
+
counts: {
|
|
612
|
+
objectsSeen: number;
|
|
613
|
+
documentsDone: number;
|
|
614
|
+
documentsSkipped: number;
|
|
615
|
+
documentsFailed: number;
|
|
616
|
+
chunks: number;
|
|
617
|
+
};
|
|
618
|
+
/** Recent per-document failures, newest first, at most twenty. */
|
|
619
|
+
samples: {
|
|
620
|
+
ref: string;
|
|
621
|
+
message: string;
|
|
622
|
+
at: string;
|
|
623
|
+
}[];
|
|
624
|
+
error: string | null;
|
|
625
|
+
/**
|
|
626
|
+
* Why a running job stopped: the owner, the budget, repeated batch
|
|
627
|
+
* failures, (mapped sources) a skip share far above the plan's, or
|
|
628
|
+
* (sources with a reader) a reader failing on far more documents than it
|
|
629
|
+
* did at first.
|
|
630
|
+
*/
|
|
631
|
+
pauseReason: 'user' | 'budget' | 'errors' | 'skips' | 'reader' | null;
|
|
632
|
+
stalled: boolean;
|
|
633
|
+
createdAt: string;
|
|
634
|
+
startedAt: string | null;
|
|
635
|
+
finishedAt: string | null;
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* One entry of a source's dictionary: your own id for a thing and the names
|
|
639
|
+
* it goes by, as your tables have them. A projection of your data, pushed
|
|
640
|
+
* whole after each load; the platform never reads your database.
|
|
641
|
+
*/
|
|
642
|
+
export interface DictionaryEntry {
|
|
643
|
+
/** An entity id: 1–128 printable ASCII, no whitespace or commas. */
|
|
644
|
+
id: string;
|
|
645
|
+
/**
|
|
646
|
+
* 1–16 names, each up to 256 characters: a tag number, a model name, an
|
|
647
|
+
* alias. Matched however a tag's separators and case are written, as whole
|
|
648
|
+
* words; a name needs three letters or digits (two in Chinese, Japanese or
|
|
649
|
+
* Korean).
|
|
650
|
+
*/
|
|
651
|
+
names: string[];
|
|
652
|
+
}
|
|
653
|
+
/** A committed dictionary version: what a push reports, and what a source reads back. */
|
|
654
|
+
export interface DictionaryReport {
|
|
655
|
+
version: number;
|
|
656
|
+
/** The matching rule its names were keyed under, carried from push to push. */
|
|
657
|
+
linkerVersion: number;
|
|
658
|
+
entries: number;
|
|
659
|
+
names: number;
|
|
660
|
+
/** Names more than one id claims: a passage naming one is tagged with every claimant. */
|
|
661
|
+
ambiguousNames: number;
|
|
662
|
+
/**
|
|
663
|
+
* The names the most ids claim: `name` as you pushed it (one of them, when
|
|
664
|
+
* several fold alike), `key` as the linker matches it (folded, separators
|
|
665
|
+
* dropped), and how many ids claim it.
|
|
666
|
+
*/
|
|
667
|
+
mostClaimed: {
|
|
668
|
+
key: string;
|
|
669
|
+
name?: string;
|
|
670
|
+
ids: number;
|
|
671
|
+
}[];
|
|
672
|
+
/**
|
|
673
|
+
* Names that can't match their own text as written, since a gap in them
|
|
674
|
+
* doesn't join (`Line 3, Unit 2`): fix these in your source data.
|
|
675
|
+
*/
|
|
676
|
+
unmatchable: {
|
|
677
|
+
count: number;
|
|
678
|
+
examples: string[];
|
|
679
|
+
};
|
|
680
|
+
committedAt: string;
|
|
681
|
+
}
|
|
682
|
+
/** What `pushDictionary` returns. */
|
|
683
|
+
export interface DictionaryPushResult extends DictionaryReport {
|
|
684
|
+
/** The dictionary was the one already committed: no new version, no relink. */
|
|
685
|
+
unchanged: boolean;
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* A relink in flight: the source's passages re-tagged against a newer
|
|
689
|
+
* dictionary version, in place, as a background walk. Searches keep working
|
|
690
|
+
* throughout; passages not yet reached carry the previous version's ids.
|
|
691
|
+
*/
|
|
692
|
+
export interface DictionaryRelink {
|
|
693
|
+
version: number;
|
|
694
|
+
lap: number;
|
|
695
|
+
/** Documents walked this lap, of `total`. */
|
|
696
|
+
walked: number;
|
|
697
|
+
total: number;
|
|
698
|
+
/** Why it's holding, when it is (the source's dedicated capacity is off). */
|
|
699
|
+
waiting: string | null;
|
|
700
|
+
error: string | null;
|
|
701
|
+
}
|
|
702
|
+
/**
|
|
703
|
+
* A typed handle to one data source. Lazy — nothing executes until a method is
|
|
704
|
+
* awaited, so it's safe to `defineDataSource()` at module scope and import the
|
|
705
|
+
* handle into route handlers (same ergonomics as `db.defineTable` and
|
|
706
|
+
* `files.defineStore`).
|
|
707
|
+
*/
|
|
708
|
+
export declare class DataSource {
|
|
709
|
+
private readonly _slug;
|
|
710
|
+
private readonly _call;
|
|
711
|
+
constructor(_slug: string, _call: DataSourcesTransport);
|
|
712
|
+
get name(): string;
|
|
713
|
+
/**
|
|
714
|
+
* Search the corpus.
|
|
715
|
+
*
|
|
716
|
+
* Returns chunks ranked by relevance, each with a citation. Searching a
|
|
717
|
+
* source that doesn't exist yet returns no results rather than throwing —
|
|
718
|
+
* code may name a corpus the build hasn't populated.
|
|
719
|
+
*
|
|
720
|
+
* **Deterministic** for a fixed corpus and configuration: the same query
|
|
721
|
+
* returns the same hits in the same order, so it's safe to build an eval set
|
|
722
|
+
* or a regression check on top of it. There is no seed to set. Two things do
|
|
723
|
+
* legitimately move the results: adding or removing documents, and changing
|
|
724
|
+
* the corpus configuration — both of which you control.
|
|
725
|
+
*
|
|
726
|
+
* Alongside `results` and `latencyMs` comes {@link SearchRan} — what the
|
|
727
|
+
* search actually did, which is the first thing to check when results don't
|
|
728
|
+
* look like the options you passed.
|
|
729
|
+
*
|
|
730
|
+
* **Throws `index_warming`** (HTTP 503) when a large corpus's index is being
|
|
731
|
+
* reloaded after an eviction: the platform keeps a working set of indexes
|
|
732
|
+
* resident and reloads the rest on first use, which for a big source takes a
|
|
733
|
+
* minute or two in the background. Treat it as "loading", never as "empty" —
|
|
734
|
+
* tell the user the knowledge base is warming up and retry shortly. Small
|
|
735
|
+
* corpora reload inside the search and never raise it. Sources on dedicated
|
|
736
|
+
* capacity are never evicted, so they never raise it either; they raise
|
|
737
|
+
* `capacity_hibernated` and friends instead when their capacity is parked.
|
|
738
|
+
*
|
|
739
|
+
* **Throws `index_building`** (HTTP 503) on dedicated capacity while a bulk
|
|
740
|
+
* load runs and after one finishes: the index is not built behind every
|
|
741
|
+
* write during a load, it is built once at the end, and until then a search
|
|
742
|
+
* would scan the corpus unindexed. The message carries "N of M vectors
|
|
743
|
+
* indexed". Handle it like `index_warming`: the knowledge base is being
|
|
744
|
+
* built, never empty; say so and retry later.
|
|
745
|
+
*/
|
|
746
|
+
search(query: string, options?: SearchOptions): Promise<{
|
|
747
|
+
results: SearchHit[];
|
|
748
|
+
mode?: SearchRan;
|
|
749
|
+
latencyMs: number;
|
|
750
|
+
}>;
|
|
751
|
+
/**
|
|
752
|
+
* Mint an ABSOLUTE, signed link to one document that works with **no**
|
|
753
|
+
* session — the same verb as `store.shareUrl()` for a file. Takes a
|
|
754
|
+
* document id or anything carrying one (`hit.citation`). Expires (default
|
|
755
|
+
* 24h; clamped to 60s–90d); each click still redirects to a fresh
|
|
756
|
+
* short-lived storage URL, so the expiry you pass is the only one that
|
|
757
|
+
* matters. For every citation of a search at once, pass
|
|
758
|
+
* {@link SearchOptions.shareCitations} instead.
|
|
759
|
+
*/
|
|
760
|
+
shareUrl(document: string | {
|
|
761
|
+
documentId: string;
|
|
762
|
+
}, options?: {
|
|
763
|
+
expiresIn?: number;
|
|
764
|
+
}): Promise<string>;
|
|
765
|
+
/**
|
|
766
|
+
* What is in the corpus, and how it was built.
|
|
767
|
+
*
|
|
768
|
+
* Document and chunk counts, storage, and the embedding model and chunking
|
|
769
|
+
* settings actually in effect — which is not the same as the platform
|
|
770
|
+
* default, since a corpus keeps the configuration it was built with until
|
|
771
|
+
* someone migrates it.
|
|
772
|
+
*/
|
|
773
|
+
stats(): Promise<DataSourceStats>;
|
|
774
|
+
/**
|
|
775
|
+
* How many passages (chunks) match a filter — exact, over the whole corpus.
|
|
776
|
+
*
|
|
777
|
+
* The honest number to show beside a search's hits: `count({ contains:
|
|
778
|
+
* query })` is how many passages contain **all** the query's words, so a UI
|
|
779
|
+
* can say "8 of 3,891 passages mention these words" and mean it. Same
|
|
780
|
+
* grammar as {@link SearchOptions.filter}, so it also answers "how many
|
|
781
|
+
* passages are tagged `department: 'legal'`" or contain an exact `phrase`.
|
|
782
|
+
* No filter counts the whole corpus.
|
|
783
|
+
*
|
|
784
|
+
* What it is not: a relevance count. Similarity is a continuous score over
|
|
785
|
+
* every chunk and no such total exists, so do not label this "relevant
|
|
786
|
+
* results" or "matches for your query". It is also not the set `search`
|
|
787
|
+
* returns: the semantic branch surfaces passages without the words, and the
|
|
788
|
+
* lexical branch scores partial overlap. Chunks, not documents — a long
|
|
789
|
+
* document contributes several.
|
|
790
|
+
*
|
|
791
|
+
* Counts share the search rate limit (300/minute per app); a count paired
|
|
792
|
+
* with every search spends two. Cheap otherwise: one index lookup, no model
|
|
793
|
+
* call. Shows the same `index_warming` / `index_building` as search while a
|
|
794
|
+
* corpus is loading or building, never a false `0`.
|
|
795
|
+
*/
|
|
796
|
+
count(filter?: SearchFilter): Promise<{
|
|
797
|
+
chunks: number;
|
|
798
|
+
latencyMs: number;
|
|
799
|
+
}>;
|
|
800
|
+
/**
|
|
801
|
+
* Which entities the passages matching a filter are about, most first.
|
|
802
|
+
*
|
|
803
|
+
* Filter on one entity (`{ entities: ['P-101A'] }`) and the answer is what
|
|
804
|
+
* appears beside it: a co-mention graph for free. Filter on a date range and
|
|
805
|
+
* it is who or what was written about then. No filter covers the corpus.
|
|
806
|
+
* `chunks` is how many matching passages carry each id, not documents: a
|
|
807
|
+
* long document's passages each count. Approximate counts, ordered most
|
|
808
|
+
* first; `limit` defaults to 20, at most 1,000. One document's entities:
|
|
809
|
+
* `{ filter: { documentIds: [id] } }`.
|
|
810
|
+
*
|
|
811
|
+
* Shares the search rate limit (300/minute per app), and shows the same
|
|
812
|
+
* `index_warming` / `index_building` as search while a corpus loads.
|
|
813
|
+
*/
|
|
814
|
+
entities(options?: {
|
|
815
|
+
filter?: SearchFilter;
|
|
816
|
+
limit?: number;
|
|
817
|
+
}): Promise<{
|
|
818
|
+
entities: {
|
|
819
|
+
id: string;
|
|
820
|
+
chunks: number;
|
|
821
|
+
}[];
|
|
822
|
+
latencyMs: number;
|
|
823
|
+
}>;
|
|
824
|
+
/**
|
|
825
|
+
* Push this source's dictionary: every entity id you have and the names it
|
|
826
|
+
* goes by, replacing the dictionary it had. From then on every passage is
|
|
827
|
+
* tagged with the ids whose names its text mentions, beside the ones its
|
|
828
|
+
* document asserts, so `filter.entities` and {@link entities} find a tag
|
|
829
|
+
* wherever it's written. Passages indexed before are re-tagged in the
|
|
830
|
+
* background (a relink; see {@link dictionary}), and nothing is
|
|
831
|
+
* re-embedded.
|
|
832
|
+
*
|
|
833
|
+
* Push it whole after each load of the tables it comes from: an unchanged
|
|
834
|
+
* dictionary commits nothing (`unchanged: true`). Takes an array or any
|
|
835
|
+
* (async) iterable, and sends it in pages, so a register of a quarter of a
|
|
836
|
+
* million entries is fine. A repeated id merges its names.
|
|
837
|
+
*/
|
|
838
|
+
pushDictionary(entries: Iterable<DictionaryEntry> | AsyncIterable<DictionaryEntry>, options?: {
|
|
839
|
+
linkerVersion?: number;
|
|
840
|
+
}): Promise<DictionaryPushResult>;
|
|
841
|
+
/**
|
|
842
|
+
* The source's current dictionary, or null when none was pushed, and the
|
|
843
|
+
* relink re-tagging its passages while one runs.
|
|
844
|
+
*/
|
|
845
|
+
dictionary(): Promise<{
|
|
846
|
+
dictionary: DictionaryReport | null;
|
|
847
|
+
relink: DictionaryRelink | null;
|
|
848
|
+
}>;
|
|
849
|
+
/**
|
|
850
|
+
* What the current dictionary finds in a piece of text: each mention, in
|
|
851
|
+
* order, with the ids it tags. How to check a dictionary before relying on
|
|
852
|
+
* it. `version` is null when the source has none. Shares the search rate
|
|
853
|
+
* limit.
|
|
854
|
+
*/
|
|
855
|
+
link(text: string): Promise<{
|
|
856
|
+
version: number | null;
|
|
857
|
+
mentions: {
|
|
858
|
+
ids: string[];
|
|
859
|
+
text: string;
|
|
860
|
+
}[];
|
|
861
|
+
}>;
|
|
862
|
+
/**
|
|
863
|
+
* Every chunk of one document, exactly as it was indexed.
|
|
864
|
+
*
|
|
865
|
+
* The direct answer to "why isn't this document coming back?" — search only
|
|
866
|
+
* shows you the chunks that surface, which is no help when none do. Reading
|
|
867
|
+
* how a document was actually split usually is.
|
|
868
|
+
*
|
|
869
|
+
* Pass `{ vectors: true }` to include each chunk's embedding. Large: roughly
|
|
870
|
+
* 8KB per chunk, so a 500-chunk document is several megabytes.
|
|
871
|
+
*/
|
|
872
|
+
chunks(documentId: string, options?: {
|
|
873
|
+
vectors?: boolean;
|
|
874
|
+
}): Promise<DataSourceChunk[]>;
|
|
875
|
+
/**
|
|
876
|
+
* Add a document to the corpus.
|
|
877
|
+
*
|
|
878
|
+
* Returns as soon as the document is queued — extraction and embedding run
|
|
879
|
+
* in the background and take a while, so poll {@link documents} for status
|
|
880
|
+
* rather than assuming the content is searchable on return.
|
|
881
|
+
*
|
|
882
|
+
* **Adding the same bytes twice is free.** Documents are content-addressed,
|
|
883
|
+
* so a re-add is a no-op when this source has already processed those exact
|
|
884
|
+
* bytes under its current configuration. Reconfiguring the source is what
|
|
885
|
+
* makes a re-add do work again — and that is an explicit, owner-triggered
|
|
886
|
+
* migration, never something a deploy causes.
|
|
887
|
+
*
|
|
888
|
+
* **On a source with a mapper** (`dataSources[].mapper` in mindstudio.json,
|
|
889
|
+
* see `defineMapper`) the bytes are a raw object and the mapper decides
|
|
890
|
+
* what they become: `documents` holds everything it produced (one object
|
|
891
|
+
* can become many documents), `document` is the first of them for
|
|
892
|
+
* convenience, and `outcome` says which way the mapper went. A
|
|
893
|
+
* `passthrough` is one document from the bytes themselves; a `deletes`
|
|
894
|
+
* outcome removes the named documents and returns none. A document whose
|
|
895
|
+
* `externalId` (or `replaces`) names an existing one supersedes it as soon
|
|
896
|
+
* as the new one is registered, before its build lands; a search in that
|
|
897
|
+
* window sees neither. **Throws `mapper_skipped`** (HTTP 422) when the
|
|
898
|
+
* mapper skipped the object — a specific thing it refused, which the caller
|
|
899
|
+
* should see — and `mapper_error` when the mapper threw, or when none of
|
|
900
|
+
* its documents passed validation. A document that fails is left out and
|
|
901
|
+
* reported in {@link AddResult.invalid}; the rest are registered.
|
|
902
|
+
*
|
|
903
|
+
* An object that maps to more than a thousand documents (a large export,
|
|
904
|
+
* streamed out of the mapper as it parses) registers its first thousand
|
|
905
|
+
* here and returns {@link AddResult.job}, a job that registers and builds
|
|
906
|
+
* the rest; `documentCount` says how many there are in all. Another job on
|
|
907
|
+
* the source refuses it with `data_source_busy`.
|
|
908
|
+
*/
|
|
909
|
+
add(content: Buffer | Uint8Array | string, options: AddOptions): Promise<AddResult>;
|
|
910
|
+
/**
|
|
911
|
+
* Documents in the corpus, with ingest status.
|
|
912
|
+
*
|
|
913
|
+
* Without options this is the first page only: the oldest thousand, enough
|
|
914
|
+
* to show a small corpus. To walk a corpus of any size use
|
|
915
|
+
* {@link allDocuments} (or {@link documentsPage} for one page at a time). To
|
|
916
|
+
* follow what {@link add} just queued, pass the ids it returned:
|
|
917
|
+
* `documents({ ids })` answers only those, at most two hundred per call.
|
|
918
|
+
*/
|
|
919
|
+
documents(options?: {
|
|
920
|
+
/** Only these documents (at most 200). */
|
|
921
|
+
ids?: string[];
|
|
922
|
+
}): Promise<DataSourceDocument[]>;
|
|
923
|
+
/**
|
|
924
|
+
* One page of the corpus with the cursor for the next page. Oldest first
|
|
925
|
+
* unless `order: 'newest'` — the page an app shows as "what just arrived"
|
|
926
|
+
* without walking anything. `limit` is at most 1,000 (the default).
|
|
927
|
+
*/
|
|
928
|
+
documentsPage(options?: {
|
|
929
|
+
cursor?: string | null;
|
|
930
|
+
limit?: number;
|
|
931
|
+
order?: 'oldest' | 'newest';
|
|
932
|
+
}): Promise<DataSourceDocumentPage>;
|
|
933
|
+
/**
|
|
934
|
+
* One page of the passages matching a filter, unranked: every page that
|
|
935
|
+
* mentions an asset (`{ entities: ['P-1004A'] }`), every passage holding a
|
|
936
|
+
* phrase, every chunk of some documents. Where {@link search} ranks and
|
|
937
|
+
* stops at its top results and {@link count} gives a number, this lists
|
|
938
|
+
* them all. Passages come document by document, each document's in order;
|
|
939
|
+
* pass `nextCursor` back for the next page, which is null after the last.
|
|
940
|
+
* `limit` defaults to 50, at most 200.
|
|
941
|
+
*
|
|
942
|
+
* Shares the search rate limit (300/minute per app), and shows the same
|
|
943
|
+
* `index_warming` / `index_building` as search while a corpus loads.
|
|
944
|
+
*/
|
|
945
|
+
passages(options?: {
|
|
946
|
+
filter?: SearchFilter;
|
|
947
|
+
cursor?: string | null;
|
|
948
|
+
limit?: number;
|
|
949
|
+
}): Promise<DataSourcePassagePage>;
|
|
950
|
+
/**
|
|
951
|
+
* Every passage matching a filter, a page at a time behind the scenes: the
|
|
952
|
+
* walk over {@link passages}. Each page counts against the search rate
|
|
953
|
+
* limit, so a large walk belongs in a background task.
|
|
954
|
+
*
|
|
955
|
+
* ```ts
|
|
956
|
+
* for await (const passage of Manuals.allPassages({ entities: ['P-1004A'] })) {
|
|
957
|
+
* mentions.push({ asset: 'P-1004A', ...passage.citation });
|
|
958
|
+
* }
|
|
959
|
+
* ```
|
|
960
|
+
*/
|
|
961
|
+
allPassages(filter?: SearchFilter, options?: {
|
|
962
|
+
/** Passages per page, at most 200 (the default). */
|
|
963
|
+
pageSize?: number;
|
|
964
|
+
}): AsyncGenerator<DataSourcePassage, void, undefined>;
|
|
965
|
+
/**
|
|
966
|
+
* Every document in the corpus, oldest first, a page at a time behind the
|
|
967
|
+
* scenes. This is how an app builds its own view of a large corpus (a
|
|
968
|
+
* timeline, counts by year, a table of ids) after an ingest: walk it once,
|
|
969
|
+
* then keep up with what each sync adds. Millions of documents is thousands
|
|
970
|
+
* of pages, so run it in a background task, not a request.
|
|
971
|
+
*
|
|
972
|
+
* ```ts
|
|
973
|
+
* for await (const doc of Archive.allDocuments()) {
|
|
974
|
+
* await index.upsert({ id: doc.id, year: doc.metadata?.year });
|
|
975
|
+
* }
|
|
976
|
+
* ```
|
|
977
|
+
*/
|
|
978
|
+
allDocuments(options?: {
|
|
979
|
+
/** Documents per page, at most 1,000 (the default). */
|
|
980
|
+
pageSize?: number;
|
|
981
|
+
}): AsyncGenerator<DataSourceDocument, void, undefined>;
|
|
982
|
+
/** Remove a document and its vectors. */
|
|
983
|
+
remove(documentId: string): Promise<void>;
|
|
984
|
+
/**
|
|
985
|
+
* Remove every document matching a selector — by metadata tags, filename,
|
|
986
|
+
* ids, or the external id prefix a bulk load or connector recorded.
|
|
987
|
+
*
|
|
988
|
+
* Works in pages of a thousand and keeps going until nothing matches (or a
|
|
989
|
+
* very large removal has run for a hundred pages, in which case `remaining`
|
|
990
|
+
* says how many are left and a second call continues). Vectors, the catalog
|
|
991
|
+
* row and the stored bytes all go; the removal is immediate and shared
|
|
992
|
+
* across dev and prod like every data-source write.
|
|
993
|
+
*
|
|
994
|
+
* ```ts
|
|
995
|
+
* await Policies.removeWhere({ metadata: { department: 'legal', year: 2019 } });
|
|
996
|
+
* await Policies.removeWhere({ externalIdPrefix: 'archive/2019/' });
|
|
997
|
+
* ```
|
|
998
|
+
*/
|
|
999
|
+
removeWhere(selector: DocumentSelector): Promise<{
|
|
1000
|
+
deleted: number;
|
|
1001
|
+
remaining: number;
|
|
1002
|
+
}>;
|
|
1003
|
+
/**
|
|
1004
|
+
* Bring the source up to date with the S3 bucket it is connected to.
|
|
1005
|
+
*
|
|
1006
|
+
* Lists the bucket, diffs it against what was ingested before (by ETag), and
|
|
1007
|
+
* runs an ingestion job over what is new or changed; keys that disappeared
|
|
1008
|
+
* upstream are removed at the end when the connector's deletion policy says
|
|
1009
|
+
* so. Returns as soon as the job is started — it plans, then runs in the
|
|
1010
|
+
* background. Poll {@link job} for progress, or read the latest sync from
|
|
1011
|
+
* {@link jobs}.
|
|
1012
|
+
*
|
|
1013
|
+
* The job auto-approves under the connector's per-sync budget. A plan over
|
|
1014
|
+
* that budget, or over the capacity of where the source lives, waits in
|
|
1015
|
+
* `planned` for a human (`remy-admin datasources jobs approve <id>`) — so the
|
|
1016
|
+
* first backfill of a large bucket stops for approval by itself, and the
|
|
1017
|
+
* steady-state deltas run unattended. Nothing is spent when nothing changed.
|
|
1018
|
+
*
|
|
1019
|
+
* The connection itself — which bucket, which app secrets hold the keys — is
|
|
1020
|
+
* created by the app's owner with `remy-admin datasources connect`, never
|
|
1021
|
+
* from code. A nightly sync is an ordinary cron interface job whose method
|
|
1022
|
+
* calls this:
|
|
1023
|
+
*
|
|
1024
|
+
* ```ts
|
|
1025
|
+
* // methods/sync-archive.ts, scheduled "0 3 * * *" in the cron interface
|
|
1026
|
+
* export default async function () {
|
|
1027
|
+
* await Archive.sync();
|
|
1028
|
+
* }
|
|
1029
|
+
* ```
|
|
1030
|
+
*
|
|
1031
|
+
* @throws `connector_not_found` when the source is not connected;
|
|
1032
|
+
* `data_source_busy` while another job or a move is in flight.
|
|
1033
|
+
*/
|
|
1034
|
+
sync(options?: {
|
|
1035
|
+
/** Override the connector's per-sync budget for this run, in dollars. */
|
|
1036
|
+
budgetDollars?: number;
|
|
1037
|
+
/**
|
|
1038
|
+
* Embed at the provider's priority tier for this run: calls are admitted
|
|
1039
|
+
* ahead of the provider's queue at 1.5x the embedding price. For the sync
|
|
1040
|
+
* where the clock matters more than the price; off by default.
|
|
1041
|
+
*/
|
|
1042
|
+
priority?: boolean;
|
|
1043
|
+
}): Promise<{
|
|
1044
|
+
job: DataSourceJob;
|
|
1045
|
+
}>;
|
|
1046
|
+
/** Recent ingestion jobs on this source (syncs and CLI loads), newest first. */
|
|
1047
|
+
jobs(): Promise<DataSourceJob[]>;
|
|
1048
|
+
/** One job by id — progress, plan, failures, and why it paused if it did. */
|
|
1049
|
+
job(id: string): Promise<DataSourceJob>;
|
|
1050
|
+
/**
|
|
1051
|
+
* Create the data source if it doesn't exist yet.
|
|
1052
|
+
*
|
|
1053
|
+
* Rarely needed — `add` and `search` both handle a missing source. Useful
|
|
1054
|
+
* when you want it to exist (and appear in the dashboard) before any
|
|
1055
|
+
* document has been added.
|
|
1056
|
+
*/
|
|
1057
|
+
ensure(name?: string): Promise<void>;
|
|
1058
|
+
/**
|
|
1059
|
+
* @internal A mapper frame's grant to store the next part of its spill: how
|
|
1060
|
+
* `defineMapper`'s executor hands the platform an object's documents a part
|
|
1061
|
+
* at a time once they outgrow the frame's record. Only a mapper frame's
|
|
1062
|
+
* token is granted one.
|
|
1063
|
+
*/
|
|
1064
|
+
_mapSpillGrant(spillId: string): Promise<{
|
|
1065
|
+
part: number;
|
|
1066
|
+
uploadUrl: string;
|
|
1067
|
+
uploadFields: Record<string, string>;
|
|
1068
|
+
}>;
|
|
1069
|
+
/**
|
|
1070
|
+
* @internal Content hash of some bytes, matching what the server computes.
|
|
1071
|
+
* Exposed for callers that want to check whether they already added a file.
|
|
1072
|
+
*/
|
|
1073
|
+
static contentHash(content: Buffer | Uint8Array | string): string;
|
|
1074
|
+
}
|