@mindstudio-ai/agent 0.1.176 → 0.1.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/analytics/index.d.ts +384 -0
- package/dist/auth/index.d.ts +113 -0
- package/dist/browser/index.d.ts +332 -0
- package/dist/cli.js +29 -29
- package/dist/client.d.ts +767 -0
- package/dist/config.d.ts +13 -0
- package/dist/context.d.ts +125 -0
- package/dist/datasources/index.d.ts +52 -0
- package/dist/datasources/source.d.ts +1074 -0
- package/dist/db/aggregate.d.ts +101 -0
- package/dist/db/index.d.ts +280 -0
- package/dist/db/mutation.d.ts +65 -0
- package/dist/db/predicate.d.ts +85 -0
- package/dist/db/query.d.ts +233 -0
- package/dist/db/raw.d.ts +36 -0
- package/dist/db/readBatching.d.ts +36 -0
- package/dist/db/scanHints.d.ts +8 -0
- package/dist/db/sql.d.ts +136 -0
- package/dist/db/table.d.ts +146 -0
- package/dist/db/time.d.ts +17 -0
- package/dist/db/types.d.ts +237 -0
- package/dist/db-ws.d.ts +35 -0
- package/dist/email/index.d.ts +396 -0
- package/dist/errors.d.ts +34 -0
- package/dist/events/index.d.ts +127 -0
- package/dist/files/index.d.ts +62 -0
- package/dist/files/store.d.ts +244 -0
- package/dist/generated/metadata.d.ts +8 -0
- package/dist/generated/snippets.d.ts +8 -0
- package/dist/generated/steps.d.ts +2745 -0
- package/dist/generated/types.d.ts +4234 -0
- package/dist/http.d.ts +22 -0
- package/dist/index.d.ts +59 -13329
- package/dist/index.js +17 -17
- package/dist/index.js.map +7 -1
- package/dist/jewel/index.d.ts +325 -0
- package/dist/mapper/index.d.ts +229 -0
- package/dist/rate-limit.d.ts +25 -0
- package/dist/reader/index.d.ts +170 -0
- package/dist/respond.d.ts +68 -0
- package/dist/task/index.d.ts +18 -0
- package/dist/task/local.d.ts +48 -0
- package/dist/task/schema.d.ts +120 -0
- package/dist/task/types.d.ts +203 -0
- package/dist/types.d.ts +591 -0
- package/dist/voice/index.d.ts +87 -0
- package/llms.txt +2 -2
- package/package.json +6 -12
- package/dist/postinstall.js +0 -12115
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jewels — agentic shadow companions for app methods.
|
|
3
|
+
*
|
|
4
|
+
* A jewel lives in `foo.jewel.ts` beside the method `foo.ts` it shadows. It
|
|
5
|
+
* proposes what a careful teammate would have done — an input for the method
|
|
6
|
+
* — without ever applying it: the method stays the only door for writes.
|
|
7
|
+
*
|
|
8
|
+
* ```ts
|
|
9
|
+
* // updateIssue.jewel.ts
|
|
10
|
+
* export default defineJewel(updateIssue, {
|
|
11
|
+
* subject: ({ issueId }) => ({ issueId }),
|
|
12
|
+
* propose: async ({ issueId }) => {
|
|
13
|
+
* // arbitrary TS: reads via plain imports, model calls via runTask
|
|
14
|
+
* return { input: { issueId, status: 'triaged' }, reasoning: '...' };
|
|
15
|
+
* },
|
|
16
|
+
* });
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* `defineJewel` returns a CALLABLE — the executor — with the config attached
|
|
20
|
+
* as properties (the Express-app pattern). That shape is deliberate: the
|
|
21
|
+
* platform's sandbox worker invokes one exported function per execution
|
|
22
|
+
* frame (`mod[handlerName](params)`), so a jewel run is an ordinary
|
|
23
|
+
* method-execution frame with zero worker or protocol changes. Dev tooling
|
|
24
|
+
* calls the same function, so dev and production share one executor body,
|
|
25
|
+
* versioned here in the SDK.
|
|
26
|
+
*
|
|
27
|
+
* Three run modes, discriminated by which key the caller provides:
|
|
28
|
+
* - `{ humanInput }` — shadow mode. The subject is derived via the jewel's
|
|
29
|
+
* projection (the human's decision fields never reach `propose`), and
|
|
30
|
+
* `humanInput` doubles as ground truth for grading.
|
|
31
|
+
* - `{ subject }` — eval / arrival mode. No human action exists yet, so the
|
|
32
|
+
* record is ungraded (an arrival proposal is graded later, when the human
|
|
33
|
+
* acts — see the grade mode below).
|
|
34
|
+
* - `{ grade: { proposed, actual } }` — grade-only mode: deferred grading of
|
|
35
|
+
* an earlier arrival proposal against the human's eventual action. Runs
|
|
36
|
+
* the jewel's own `grade` (or the default deep-equal) and nothing else.
|
|
37
|
+
*
|
|
38
|
+
* The executor NEVER throws: a shadow run must never break anything. Author
|
|
39
|
+
* code failing (`subject`/`propose`) is captured in the record's `error`;
|
|
40
|
+
* `grade` failing softens to verdict `'skip'`.
|
|
41
|
+
*/
|
|
42
|
+
/** Any app method a jewel can shadow: one JSON-serializable input, any result. */
|
|
43
|
+
export type JewelMethod = (input: any) => any;
|
|
44
|
+
/**
|
|
45
|
+
* The input type of the shadowed method. Guarded so a zero-parameter method
|
|
46
|
+
* resolves to `undefined` instead of erroring on `Parameters<M>[0]`.
|
|
47
|
+
*/
|
|
48
|
+
export type JewelMethodInput<M extends JewelMethod> = Parameters<M> extends [] ? undefined : Parameters<M>[0];
|
|
49
|
+
/**
|
|
50
|
+
* What `propose` returns. `input: null` is abstention — a first-class,
|
|
51
|
+
* graded outcome, not an error. Reasoning is most valuable on abstention.
|
|
52
|
+
*/
|
|
53
|
+
export interface JewelProposal<I> {
|
|
54
|
+
input: I | null;
|
|
55
|
+
reasoning: string;
|
|
56
|
+
/**
|
|
57
|
+
* Transcript id(s) of the runTask call(s) that constitute this decision
|
|
58
|
+
* (`task.traceId`; an array for escalation chains). Attaching preserves
|
|
59
|
+
* the full model transcript with the pair — the training row and, at
|
|
60
|
+
* auto, the audit trail. Helper/formatting calls stay unattached: the
|
|
61
|
+
* attachment is the label for which run WAS the decision.
|
|
62
|
+
*/
|
|
63
|
+
trace?: string | string[];
|
|
64
|
+
}
|
|
65
|
+
export interface JewelVerdict {
|
|
66
|
+
verdict: 'agree' | 'disagree' | 'skip';
|
|
67
|
+
notes?: string;
|
|
68
|
+
/** Transcript id(s) of the judge's runTask call(s), when grade used one. */
|
|
69
|
+
trace?: string | string[];
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Argument to a custom `grade`. `proposed` is null when the jewel abstained;
|
|
73
|
+
* `actual` is always present — grading only happens in shadow mode, where
|
|
74
|
+
* the human acted.
|
|
75
|
+
*/
|
|
76
|
+
export interface JewelGradeContext<I> {
|
|
77
|
+
proposed: I | null;
|
|
78
|
+
actual: I;
|
|
79
|
+
}
|
|
80
|
+
type MaybePromise<T> = T | Promise<T>;
|
|
81
|
+
/**
|
|
82
|
+
* Authoring config. Declare `subject` before `propose` — the subject type
|
|
83
|
+
* is inferred from the projection's return and flows into `propose`'s
|
|
84
|
+
* parameter.
|
|
85
|
+
*/
|
|
86
|
+
export interface JewelConfig<M extends JewelMethod, S> {
|
|
87
|
+
/**
|
|
88
|
+
* Projection: method input → subject. What the human was looking at,
|
|
89
|
+
* never what they decided — this is what keeps the label out of the exam.
|
|
90
|
+
*/
|
|
91
|
+
subject: (input: JewelMethodInput<M>) => S;
|
|
92
|
+
/**
|
|
93
|
+
* The proposal. Runs on the projection only; arbitrary TS (plain imports
|
|
94
|
+
* for context, model calls via runTask). Throwing never propagates — it
|
|
95
|
+
* becomes an `error` on the pair record.
|
|
96
|
+
*/
|
|
97
|
+
propose: (subject: NoInfer<S>) => MaybePromise<JewelProposal<JewelMethodInput<M>>>;
|
|
98
|
+
/**
|
|
99
|
+
* The custom assertion. Omit for deep-equal on the method input. May be
|
|
100
|
+
* async and call models. Throwing softens to verdict `'skip'`.
|
|
101
|
+
*
|
|
102
|
+
* Strict-safe field iteration, when grading only touched fields:
|
|
103
|
+
* ```ts
|
|
104
|
+
* grade: ({ proposed, actual }) => {
|
|
105
|
+
* if (!proposed) return { verdict: 'disagree', notes: 'abstained' };
|
|
106
|
+
* const keys = Object.keys(actual) as (keyof typeof actual)[];
|
|
107
|
+
* const misses = keys.filter((k) => proposed[k] !== actual[k]);
|
|
108
|
+
* return misses.length
|
|
109
|
+
* ? { verdict: 'disagree', notes: misses.join(', ') }
|
|
110
|
+
* : { verdict: 'agree' };
|
|
111
|
+
* }
|
|
112
|
+
* ```
|
|
113
|
+
*/
|
|
114
|
+
grade?: (ctx: JewelGradeContext<JewelMethodInput<M>>) => MaybePromise<JewelVerdict>;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Executor params — constructed by the platform (or dev tooling), exactly
|
|
118
|
+
* one key. `?: never` on the other keys rejects passing more than one.
|
|
119
|
+
*/
|
|
120
|
+
export type JewelRunParams<I, S> = {
|
|
121
|
+
humanInput: I;
|
|
122
|
+
subject?: never;
|
|
123
|
+
grade?: never;
|
|
124
|
+
} | {
|
|
125
|
+
subject: S;
|
|
126
|
+
humanInput?: never;
|
|
127
|
+
grade?: never;
|
|
128
|
+
} | {
|
|
129
|
+
grade: JewelGradeContext<I>;
|
|
130
|
+
humanInput?: never;
|
|
131
|
+
subject?: never;
|
|
132
|
+
};
|
|
133
|
+
/**
|
|
134
|
+
* The versioned, JSON-serializable output of one jewel run — the row the
|
|
135
|
+
* pair ledger stores. Values are kept verbatim, so method inputs must be
|
|
136
|
+
* JSON-safe (they already crossed the wire as JSON in real use).
|
|
137
|
+
*/
|
|
138
|
+
export interface JewelPairRecord<I = unknown, S = unknown> {
|
|
139
|
+
v: 1;
|
|
140
|
+
/** `grade` records are consumed by the platform (verdict extracted from a
|
|
141
|
+
* grade-only run) and never persisted as pair rows themselves. */
|
|
142
|
+
mode: 'shadow' | 'eval' | 'grade';
|
|
143
|
+
/** Absent only when the projection itself threw (shadow mode). */
|
|
144
|
+
subject?: S;
|
|
145
|
+
/** null = abstention. Absent when propose failed. */
|
|
146
|
+
proposed?: I | null;
|
|
147
|
+
/** Absent when propose failed. */
|
|
148
|
+
reasoning?: string;
|
|
149
|
+
/** The human's input — present in shadow mode. */
|
|
150
|
+
actual?: I;
|
|
151
|
+
/** Present iff graded (shadow or grade mode, propose succeeded). */
|
|
152
|
+
verdict?: 'agree' | 'disagree' | 'skip';
|
|
153
|
+
notes?: string;
|
|
154
|
+
/** Present iff subject() or propose() threw. Grade errors become verdict 'skip'. */
|
|
155
|
+
error?: {
|
|
156
|
+
phase: 'subject' | 'propose';
|
|
157
|
+
message: string;
|
|
158
|
+
stack?: string;
|
|
159
|
+
};
|
|
160
|
+
/**
|
|
161
|
+
* Transcript ids attached by propose. The transcripts themselves live
|
|
162
|
+
* platform-side, recorded per turn and keyed by (run id, trace id) — the
|
|
163
|
+
* record carries only the ids.
|
|
164
|
+
*/
|
|
165
|
+
trace?: string[];
|
|
166
|
+
/** Transcript ids attached by a custom grade's verdict. */
|
|
167
|
+
gradeTrace?: string[];
|
|
168
|
+
/**
|
|
169
|
+
* Whether this jewel declares a custom `grade`. The platform's deferred
|
|
170
|
+
* grading dispatches a grade-mode run when true; when false it grades
|
|
171
|
+
* locally with an equivalent of the default deep-equal (no sandbox trip
|
|
172
|
+
* to evaluate a pure structural comparison).
|
|
173
|
+
*/
|
|
174
|
+
customGrade: boolean;
|
|
175
|
+
startedAt: number;
|
|
176
|
+
durationMs: number;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* The export of a `foo.jewel.ts` file: the executor, callable as an
|
|
180
|
+
* ordinary handler, with the authored config attached for the compiler and
|
|
181
|
+
* dev tooling. `method` carries the actual function reference — reference
|
|
182
|
+
* identity is what lets the compiler verify the manifest's method↔jewel
|
|
183
|
+
* pairing against the code. `grade` is `undefined` when the default
|
|
184
|
+
* deep-equal grade applies; presence is the custom-grade signal.
|
|
185
|
+
*/
|
|
186
|
+
export interface Jewel<M extends JewelMethod, S> {
|
|
187
|
+
(params: JewelRunParams<JewelMethodInput<M>, S>): Promise<JewelPairRecord<JewelMethodInput<M>, S>>;
|
|
188
|
+
readonly kind: 'jewel';
|
|
189
|
+
readonly method: M;
|
|
190
|
+
readonly subject: (input: JewelMethodInput<M>) => S;
|
|
191
|
+
readonly propose: (subject: S) => MaybePromise<JewelProposal<JewelMethodInput<M>>>;
|
|
192
|
+
readonly grade: ((ctx: JewelGradeContext<JewelMethodInput<M>>) => MaybePromise<JewelVerdict>) | undefined;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* What the platform did with a proposal, routed by the method's autonomy:
|
|
196
|
+
* - `recorded` — shadow: proposal written to the pair ledger, graded later
|
|
197
|
+
* against the human's eventual action (within the attribution window).
|
|
198
|
+
* - `queued` — approve: waiting in the review queue.
|
|
199
|
+
* - `committed` — auto: the jewel's proposal was applied (the method ran);
|
|
200
|
+
* `output` carries the method's return value.
|
|
201
|
+
* - `abstained` — the jewel chose not to act; recorded, still gradeable.
|
|
202
|
+
* - `disabled` — the method has no jewel or autonomy is `manual`. Returned,
|
|
203
|
+
* never thrown: dialing a method down must not break the app code that
|
|
204
|
+
* proposes to it.
|
|
205
|
+
* - `skipped` — jewel-descended recursion, unsampled (sampleRate), or a dev
|
|
206
|
+
* session whose CLI didn't report jewel metadata. Dev sessions otherwise
|
|
207
|
+
* run the full behavioral flow (queue items scoped to the session, auto
|
|
208
|
+
* commits against the dev database) without recording any training data.
|
|
209
|
+
* - `failed` — an auto commit was attempted and the method rejected it
|
|
210
|
+
* (e.g. the state was consumed concurrently). The moment stays pending.
|
|
211
|
+
* - `pending` — a concurrent replay: the original propose for this
|
|
212
|
+
* idempotencyKey is still mid-run. Treat as accepted.
|
|
213
|
+
*/
|
|
214
|
+
export type JewelProposeOutcome = 'recorded' | 'queued' | 'committed' | 'abstained' | 'disabled' | 'skipped' | 'failed' | 'pending';
|
|
215
|
+
export interface JewelProposeResult {
|
|
216
|
+
outcome: JewelProposeOutcome;
|
|
217
|
+
/** Present on `committed` — the method's return value. */
|
|
218
|
+
output?: unknown;
|
|
219
|
+
/** Present on `queued` — address the item via `jewels.queue.resolve`. */
|
|
220
|
+
queueItemId?: string;
|
|
221
|
+
}
|
|
222
|
+
/** A pending approval-queue item awaiting a reviewer. */
|
|
223
|
+
export interface JewelQueueItem {
|
|
224
|
+
id: string;
|
|
225
|
+
methodId: string;
|
|
226
|
+
subject: Record<string, unknown>;
|
|
227
|
+
/** The method input the jewel proposes to apply. */
|
|
228
|
+
proposed: unknown;
|
|
229
|
+
reasoning: string | null;
|
|
230
|
+
proposedAt: string;
|
|
231
|
+
/** Unresolved items expire (verdict `expired`) at the attribution window. */
|
|
232
|
+
expiresAt: string;
|
|
233
|
+
}
|
|
234
|
+
export type JewelQueueResolution = 'approved' | 'edited' | 'dismissed';
|
|
235
|
+
export interface JewelQueueResolveResult {
|
|
236
|
+
resolution: JewelQueueResolution;
|
|
237
|
+
/** Present on approve — the applied method's return value. */
|
|
238
|
+
output?: unknown;
|
|
239
|
+
}
|
|
240
|
+
export interface JewelsApi {
|
|
241
|
+
/**
|
|
242
|
+
* Hand a decision moment to a method's jewel — the arrival-shaped trigger.
|
|
243
|
+
* Place it where the app knows the moment was born (an ingest branch that
|
|
244
|
+
* lands a row in its pending state). The platform routes by the method's
|
|
245
|
+
* autonomy; see {@link JewelProposeOutcome}.
|
|
246
|
+
*
|
|
247
|
+
* `idempotencyKey` (Stripe semantics — a replayed key returns the ORIGINAL
|
|
248
|
+
* outcome, so retried webhooks are invisible to this code) defaults to a
|
|
249
|
+
* hash of the subject. Sibling proposals for one decision moment should
|
|
250
|
+
* share a key so cross-verb grading can close them together.
|
|
251
|
+
*
|
|
252
|
+
* Backend/managed contexts only (rides the execution's hook token). Runs
|
|
253
|
+
* the jewel synchronously — wrap chains in `mindstudio.waitUntil(...)` so
|
|
254
|
+
* the calling method returns immediately:
|
|
255
|
+
*
|
|
256
|
+
* ```ts
|
|
257
|
+
* mindstudio.waitUntil((async () => {
|
|
258
|
+
* const merge = await mindstudio.jewels.propose(
|
|
259
|
+
* 'merge-issues', { sourceId: issue.id }, { idempotencyKey: issue.id });
|
|
260
|
+
* if (merge.outcome !== 'committed') {
|
|
261
|
+
* await mindstudio.jewels.propose(
|
|
262
|
+
* 'triage-issue', { issueId: issue.id }, { idempotencyKey: issue.id });
|
|
263
|
+
* }
|
|
264
|
+
* })());
|
|
265
|
+
* ```
|
|
266
|
+
*/
|
|
267
|
+
propose(methodId: string, subject: Record<string, unknown>, opts?: {
|
|
268
|
+
idempotencyKey?: string;
|
|
269
|
+
}): Promise<JewelProposeResult>;
|
|
270
|
+
/**
|
|
271
|
+
* The app-native approval queue for `approve`-mode methods. Build the
|
|
272
|
+
* review UI in the app itself: a backend method lists items (gate it with
|
|
273
|
+
* the app's reviewer role), the frontend renders the inbox, and a resolve
|
|
274
|
+
* method approves or dismisses.
|
|
275
|
+
*/
|
|
276
|
+
queue: {
|
|
277
|
+
/** Pending items, oldest first. */
|
|
278
|
+
list(opts?: {
|
|
279
|
+
methodId?: string;
|
|
280
|
+
limit?: number;
|
|
281
|
+
}): Promise<{
|
|
282
|
+
items: JewelQueueItem[];
|
|
283
|
+
}>;
|
|
284
|
+
/**
|
|
285
|
+
* Resolve one item. `approve` APPLIES the target method as the current
|
|
286
|
+
* session user — the reviewer — so the effect belongs to the human who
|
|
287
|
+
* clicked, and the target method's own auth checks are the real gate on
|
|
288
|
+
* who may approve. Pass `input` to apply an edited version of the
|
|
289
|
+
* proposal (captured as resolution `edited` — proposed/edited/final ride
|
|
290
|
+
* the pair record; the richest training signal). `dismiss` records the
|
|
291
|
+
* rejection and closes the item without running anything; other verbs'
|
|
292
|
+
* proposals for the same moment stay open.
|
|
293
|
+
*/
|
|
294
|
+
resolve(itemId: string, opts: {
|
|
295
|
+
action: 'approve' | 'dismiss';
|
|
296
|
+
input?: Record<string, unknown>;
|
|
297
|
+
}): Promise<JewelQueueResolveResult>;
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Define a jewel — the agentic shadow companion for an app method.
|
|
302
|
+
*
|
|
303
|
+
* @example
|
|
304
|
+
* ```ts
|
|
305
|
+
* export default defineJewel(updateIssue, {
|
|
306
|
+
* subject: ({ issueId }) => ({ issueId }),
|
|
307
|
+
* propose: async ({ issueId }) => {
|
|
308
|
+
* const issue = await resolveIssue(issueId);
|
|
309
|
+
* if (!issue || issue.status !== 'new') {
|
|
310
|
+
* return { input: null, reasoning: 'Nothing to propose.' };
|
|
311
|
+
* }
|
|
312
|
+
* // ...gather context, call a model via runTask + outputSchema...
|
|
313
|
+
* return { input: { issueId, status: 'triaged' }, reasoning: '...' };
|
|
314
|
+
* },
|
|
315
|
+
* grade: async ({ proposed, actual }) => {
|
|
316
|
+
* if (!proposed) return { verdict: 'disagree', notes: 'abstained' };
|
|
317
|
+
* return proposed.status === actual.status
|
|
318
|
+
* ? { verdict: 'agree' }
|
|
319
|
+
* : { verdict: 'disagree' };
|
|
320
|
+
* },
|
|
321
|
+
* });
|
|
322
|
+
* ```
|
|
323
|
+
*/
|
|
324
|
+
export declare function defineJewel<M extends JewelMethod, S>(method: M, config: JewelConfig<M, S>): Jewel<M, S>;
|
|
325
|
+
export {};
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mappers — customer code that turns raw objects into documents.
|
|
3
|
+
*
|
|
4
|
+
* A data source takes every object it is given as one document through
|
|
5
|
+
* built-in extraction. When the file is not the document — a JSON record
|
|
6
|
+
* that should become markdown plus metadata, a JSONL object that holds a
|
|
7
|
+
* thousand articles, a kill notice that means "remove this story" — the
|
|
8
|
+
* source declares a MAPPER: a `<slug>.mapper.ts` file beside the source's
|
|
9
|
+
* definition, exporting `defineMapper(Source, { map })`.
|
|
10
|
+
*
|
|
11
|
+
* ```ts
|
|
12
|
+
* // datasources/archive.mapper.ts
|
|
13
|
+
* export default defineMapper(Archive, {
|
|
14
|
+
* map: async (object) => {
|
|
15
|
+
* if (!object.key.endsWith('.json')) return passthrough();
|
|
16
|
+
* const { item } = await object.json();
|
|
17
|
+
* if (item.pubstatus === 'canceled') return deletes([item.uri]);
|
|
18
|
+
* return documents([{
|
|
19
|
+
* externalId: item.uri,
|
|
20
|
+
* title: item.headline,
|
|
21
|
+
* markdown: toMarkdown(item),
|
|
22
|
+
* metadata: { date: item.versioncreated.slice(0, 10), language: item.language },
|
|
23
|
+
* replaces: item.altids?.original_id,
|
|
24
|
+
* }]);
|
|
25
|
+
* },
|
|
26
|
+
* });
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* One object in, one of four outcomes out:
|
|
30
|
+
* - `documents([...])` — the object becomes these documents (an array is
|
|
31
|
+
* one object → many). Each is `{ externalId, title, markdown, metadata?,
|
|
32
|
+
* entities?, replaces? }`; `externalId` is the identity the platform
|
|
33
|
+
* replaces by. Any number: pass a generator (sync or async) to stream an
|
|
34
|
+
* export of a million rows without holding it. A document that fails
|
|
35
|
+
* validation is left out and named; the rest of the object goes in.
|
|
36
|
+
* - `passthrough({ metadata?, entities? })` — ingest the raw object itself through
|
|
37
|
+
* built-in extraction, as an unmapped source would. What lets a mapped
|
|
38
|
+
* source still accept a PDF.
|
|
39
|
+
* - `skip(reason)` — not a document. Quarantined on the job with the reason;
|
|
40
|
+
* an `add()` of a skipped object throws `mapper_skipped`.
|
|
41
|
+
* - `deletes([externalId, ...])` — the live documents with these external ids
|
|
42
|
+
* are removed.
|
|
43
|
+
* A thrown error is the object's `error` outcome: quarantined, the job goes on.
|
|
44
|
+
*
|
|
45
|
+
* `defineMapper` returns a CALLABLE — the executor — with the config attached
|
|
46
|
+
* (the same pattern as `defineJewel`). The platform runs it as an ordinary
|
|
47
|
+
* execution frame: one call maps a slice of objects and returns their
|
|
48
|
+
* outcomes, and stores the documents its objects produce as it goes, a part
|
|
49
|
+
* of up to a thousand at a time, so a frame's memory holds one part
|
|
50
|
+
* whatever its objects become. Everything a method can do, a mapper can do — call models via
|
|
51
|
+
* `runTask`, `fetch` an external API, read the app's tables — and every call
|
|
52
|
+
* is spend per object. The mapper's `map` runs under a per-object timeout
|
|
53
|
+
* (declared in `mindstudio.json`, default 30 s), which also bounds the wait
|
|
54
|
+
* for each next document a generator yields; a whole object has the frame's
|
|
55
|
+
* twenty minutes. The executor never throws.
|
|
56
|
+
*
|
|
57
|
+
* On a job, mapping is its own stage: the mapper turns each object into
|
|
58
|
+
* documents, and the platform ingests those documents in parallel batches of
|
|
59
|
+
* fifty across its workers, whatever one object became. An object's size does
|
|
60
|
+
* not set the pace; the number of objects sets how wide the mapping runs.
|
|
61
|
+
*
|
|
62
|
+
* Declared in `mindstudio.json`:
|
|
63
|
+
* ```json
|
|
64
|
+
* "dataSources": [{ "slug": "archive", "mapper": { "path": "dist/datasources/archive.mapper.ts" } }]
|
|
65
|
+
* ```
|
|
66
|
+
* Everything that enters a mapped source is mapped: bulk jobs, connector
|
|
67
|
+
* syncs, and `Source.add()` alike. Mapper fixes are re-applied with
|
|
68
|
+
* `remy-admin datasources remap` from the platform's copy of each raw object.
|
|
69
|
+
*/
|
|
70
|
+
import type { DocumentMetadata } from '../datasources/source.js';
|
|
71
|
+
/** The raw object handed to `map`. Bytes are read lazily, once. */
|
|
72
|
+
export interface MapObject {
|
|
73
|
+
/** The key the object is known by: a store key, a bucket key, a URL, a filename. */
|
|
74
|
+
key: string;
|
|
75
|
+
size: number | null;
|
|
76
|
+
contentType: string | null;
|
|
77
|
+
/** Bucket objects carry these; other origins report null. */
|
|
78
|
+
etag: string | null;
|
|
79
|
+
lastModified: string | null;
|
|
80
|
+
/** Metadata the origin supplied (a manifest line's `metadata`), or null. */
|
|
81
|
+
metadata: DocumentMetadata | null;
|
|
82
|
+
/** Entities the origin supplied (a manifest line's, an `add()`'s), or null. */
|
|
83
|
+
entities: string[] | null;
|
|
84
|
+
bytes(): Promise<Uint8Array>;
|
|
85
|
+
text(): Promise<string>;
|
|
86
|
+
json<T = unknown>(): Promise<T>;
|
|
87
|
+
}
|
|
88
|
+
export interface MappedDocument {
|
|
89
|
+
/**
|
|
90
|
+
* The identity the platform replaces by. A later object producing the same
|
|
91
|
+
* external id (a new version of the story) supersedes this document; a
|
|
92
|
+
* `deletes([...])` naming it removes it. Filterable at search time via
|
|
93
|
+
* `externalIdPrefix`.
|
|
94
|
+
*/
|
|
95
|
+
externalId: string;
|
|
96
|
+
title: string;
|
|
97
|
+
/** The document's text. Chunked and embedded as markdown. */
|
|
98
|
+
markdown: string;
|
|
99
|
+
/** Scalars only, up to 16 keys — see `AddOptions.metadata`. */
|
|
100
|
+
metadata?: DocumentMetadata;
|
|
101
|
+
/**
|
|
102
|
+
* What the document is about, as your own ids — see `AddOptions.entities`.
|
|
103
|
+
* A record that names its subject (a work order's asset, a story's people)
|
|
104
|
+
* says so here, whether or not its text does.
|
|
105
|
+
*/
|
|
106
|
+
entities?: string[];
|
|
107
|
+
/** External ids this document replaces (a writethru's original id). */
|
|
108
|
+
replaces?: string | string[];
|
|
109
|
+
}
|
|
110
|
+
/** What `documents()` takes: one document, an array, or any iterable or async iterable. */
|
|
111
|
+
export type MappedDocuments = MappedDocument | Iterable<MappedDocument> | AsyncIterable<MappedDocument>;
|
|
112
|
+
export type MapOutcome = {
|
|
113
|
+
kind: 'documents';
|
|
114
|
+
documents: Iterable<MappedDocument> | AsyncIterable<MappedDocument>;
|
|
115
|
+
} | {
|
|
116
|
+
kind: 'passthrough';
|
|
117
|
+
metadata?: DocumentMetadata;
|
|
118
|
+
entities?: string[];
|
|
119
|
+
} | {
|
|
120
|
+
kind: 'skip';
|
|
121
|
+
reason: string;
|
|
122
|
+
} | {
|
|
123
|
+
kind: 'deletes';
|
|
124
|
+
externalIds: string[];
|
|
125
|
+
};
|
|
126
|
+
type MaybePromise<T> = T | Promise<T>;
|
|
127
|
+
export interface MapperConfig {
|
|
128
|
+
/** One raw object in; an outcome out. Throwing is the object's `error` outcome. */
|
|
129
|
+
map: (object: MapObject) => MaybePromise<MapOutcome>;
|
|
130
|
+
}
|
|
131
|
+
/** One object as the platform hands it to the executor: metadata plus a read URL. */
|
|
132
|
+
export interface MapRunObject {
|
|
133
|
+
key: string;
|
|
134
|
+
size: number | null;
|
|
135
|
+
contentType: string | null;
|
|
136
|
+
etag: string | null;
|
|
137
|
+
lastModified: string | null;
|
|
138
|
+
metadata: DocumentMetadata | null;
|
|
139
|
+
entities: string[] | null;
|
|
140
|
+
/** Short-lived read URL for the object's bytes. */
|
|
141
|
+
url: string;
|
|
142
|
+
/**
|
|
143
|
+
* The object's bytes, base64, when the platform already held them (it
|
|
144
|
+
* hashes and lands every object before mapping, and small objects ride
|
|
145
|
+
* along). Present, it is the bytes; `url` is the fallback for large objects.
|
|
146
|
+
*/
|
|
147
|
+
content?: string;
|
|
148
|
+
}
|
|
149
|
+
export interface MapRunParams {
|
|
150
|
+
v: 1;
|
|
151
|
+
objects: MapRunObject[];
|
|
152
|
+
/** Per-object budget for `map`, in milliseconds. */
|
|
153
|
+
timeoutMs?: number;
|
|
154
|
+
/** Objects mapped at once inside the frame. */
|
|
155
|
+
concurrency?: number;
|
|
156
|
+
/** Where the frame's documents are stored, a part at a time. */
|
|
157
|
+
spill?: {
|
|
158
|
+
id: string;
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* One stored part of a frame's documents: gzipped NDJSON, a line `{ o, i, d }`
|
|
163
|
+
* per document (the object's index in the frame, the document's index in
|
|
164
|
+
* that object's output, the document).
|
|
165
|
+
*/
|
|
166
|
+
export interface MapRunPart {
|
|
167
|
+
part: number;
|
|
168
|
+
count: number;
|
|
169
|
+
}
|
|
170
|
+
export type MapRunOutcome = {
|
|
171
|
+
kind: 'documents';
|
|
172
|
+
count: number;
|
|
173
|
+
} | Exclude<MapOutcome, {
|
|
174
|
+
kind: 'documents';
|
|
175
|
+
}> | {
|
|
176
|
+
kind: 'error';
|
|
177
|
+
error: {
|
|
178
|
+
message: string;
|
|
179
|
+
stack?: string;
|
|
180
|
+
};
|
|
181
|
+
};
|
|
182
|
+
export interface MapRunResult {
|
|
183
|
+
key: string;
|
|
184
|
+
outcome: MapRunOutcome;
|
|
185
|
+
durationMs: number;
|
|
186
|
+
}
|
|
187
|
+
/** What one executor call returns: one result per input object, in order. */
|
|
188
|
+
export interface MapRunRecord {
|
|
189
|
+
v: 1;
|
|
190
|
+
source: string;
|
|
191
|
+
outcomes: MapRunResult[];
|
|
192
|
+
/** Every document the frame's objects produced, in the order stored. */
|
|
193
|
+
parts: MapRunPart[];
|
|
194
|
+
startedAt: number;
|
|
195
|
+
durationMs: number;
|
|
196
|
+
}
|
|
197
|
+
/** The callable executor with its config attached. */
|
|
198
|
+
export interface Mapper {
|
|
199
|
+
(params: MapRunParams): Promise<MapRunRecord>;
|
|
200
|
+
readonly kind: 'mapper';
|
|
201
|
+
/** The data source's slug. */
|
|
202
|
+
readonly source: string;
|
|
203
|
+
readonly config: MapperConfig;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* The object becomes these documents: one passed bare, an array, or a
|
|
207
|
+
* generator that yields them, which streams an export of any size.
|
|
208
|
+
*/
|
|
209
|
+
export declare function documents(list: MappedDocuments): MapOutcome;
|
|
210
|
+
/**
|
|
211
|
+
* Ingest the raw object itself through built-in extraction. `metadata` and
|
|
212
|
+
* `entities`, when given, replace what the origin supplied.
|
|
213
|
+
*/
|
|
214
|
+
export declare function passthrough(options?: {
|
|
215
|
+
metadata?: DocumentMetadata;
|
|
216
|
+
entities?: string[];
|
|
217
|
+
}): MapOutcome;
|
|
218
|
+
/** Not a document. Quarantined with the reason. */
|
|
219
|
+
export declare function skip(reason: string): MapOutcome;
|
|
220
|
+
/** Remove the live documents with these external ids. */
|
|
221
|
+
export declare function deletes(externalIds: string | string[]): MapOutcome;
|
|
222
|
+
/**
|
|
223
|
+
* Define a mapper — the code that turns each raw object entering a data
|
|
224
|
+
* source into documents. See the module doc for the four outcomes.
|
|
225
|
+
*/
|
|
226
|
+
export declare function defineMapper(source: {
|
|
227
|
+
readonly name: string;
|
|
228
|
+
}, config: MapperConfig): Mapper;
|
|
229
|
+
export {};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export type AuthType = 'internal' | 'apiKey';
|
|
2
|
+
/**
|
|
3
|
+
* Keeps an execution's calls within the server's concurrency limit, so a
|
|
4
|
+
* `Promise.all` over many steps queues here instead of being refused.
|
|
5
|
+
*
|
|
6
|
+
* The server's limits are per execution (per hook token), so the slots are
|
|
7
|
+
* too: each `runWithContext()` request gets its own, and a persistent sandbox
|
|
8
|
+
* worker's concurrent executions never wait on each other. Outside a request
|
|
9
|
+
* context (API-key use, or one execution per process) the instance holds one
|
|
10
|
+
* set. There is no call cap here: the server enforces it and answers 429 with
|
|
11
|
+
* `Retry-After`, which `http.ts` retries.
|
|
12
|
+
*/
|
|
13
|
+
export declare class RateLimiter {
|
|
14
|
+
private concurrencyLimit;
|
|
15
|
+
private perRequest;
|
|
16
|
+
private shared;
|
|
17
|
+
constructor(authType: AuthType);
|
|
18
|
+
/** Acquire a slot. Resolves when a concurrent slot is available. */
|
|
19
|
+
acquire(): Promise<void>;
|
|
20
|
+
/** Release a slot and let the next queued request proceed. */
|
|
21
|
+
release(): void;
|
|
22
|
+
/** Update limits from response headers. */
|
|
23
|
+
updateFromHeaders(headers: Headers): void;
|
|
24
|
+
private slots;
|
|
25
|
+
}
|