@madewithremy/admin 0.1.56 → 0.1.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +1 -1
- package/dist/client.d.ts +199 -0
- package/dist/config.d.ts +8 -0
- package/dist/ctx.d.ts +36 -0
- package/dist/errors.d.ts +68 -0
- package/dist/http.d.ts +68 -0
- package/dist/index.d.ts +7 -9434
- package/dist/index.js +6 -6
- package/dist/index.js.map +7 -1
- package/dist/ops/agent.d.ts +44 -0
- package/dist/ops/analytics.d.ts +285 -0
- package/dist/ops/apps.d.ts +33 -0
- package/dist/ops/crashes.d.ts +86 -0
- package/dist/ops/cron.d.ts +40 -0
- package/dist/ops/data.d.ts +68 -0
- package/dist/ops/dataSourceConnectors.d.ts +80 -0
- package/dist/ops/dataSourceEvals.d.ts +208 -0
- package/dist/ops/dataSourceJobs.d.ts +141 -0
- package/dist/ops/dataSourceMappers.d.ts +125 -0
- package/dist/ops/dataSources.d.ts +654 -0
- package/dist/ops/db.d.ts +81 -0
- package/dist/ops/diagnostics.d.ts +42 -0
- package/dist/ops/domains.d.ts +250 -0
- package/dist/ops/email.d.ts +305 -0
- package/dist/ops/events.d.ts +56 -0
- package/dist/ops/feedback.d.ts +24 -0
- package/dist/ops/files.d.ts +166 -0
- package/dist/ops/infra.d.ts +163 -0
- package/dist/ops/issues.d.ts +160 -0
- package/dist/ops/jewels.d.ts +249 -0
- package/dist/ops/methods.d.ts +60 -0
- package/dist/ops/prerender.d.ts +74 -0
- package/dist/ops/previews.d.ts +52 -0
- package/dist/ops/qaRecordings.d.ts +68 -0
- package/dist/ops/releases.d.ts +193 -0
- package/dist/ops/requests.d.ts +73 -0
- package/dist/ops/secrets.d.ts +69 -0
- package/dist/ops/settings.d.ts +45 -0
- package/dist/ops/tasks.d.ts +47 -0
- package/dist/ops/usage.d.ts +40 -0
- package/dist/ops/users.d.ts +59 -0
- package/dist/ops/voice.d.ts +178 -0
- package/dist/output.d.ts +19 -0
- package/dist/parseJsonConfig.d.ts +41 -0
- package/dist/poll.d.ts +38 -0
- package/dist/prod.js +24 -21
- package/dist/runningMethod.d.ts +25 -0
- package/dist/sleep.d.ts +12 -0
- package/dist/types/agent.d.ts +95 -0
- package/dist/types/analytics.d.ts +225 -0
- package/dist/types/apps.d.ts +39 -0
- package/dist/types/crashes.d.ts +81 -0
- package/dist/types/cron.d.ts +68 -0
- package/dist/types/data.d.ts +38 -0
- package/dist/types/dataSourceConnectors.d.ts +59 -0
- package/dist/types/dataSourceEvals.d.ts +225 -0
- package/dist/types/dataSourceJobs.d.ts +283 -0
- package/dist/types/dataSourceMappers.d.ts +204 -0
- package/dist/types/dataSourceReaders.d.ts +102 -0
- package/dist/types/dataSources.d.ts +627 -0
- package/dist/types/db.d.ts +63 -0
- package/dist/types/diagnostics.d.ts +96 -0
- package/dist/types/domains.d.ts +277 -0
- package/dist/types/email.d.ts +400 -0
- package/dist/types/events.d.ts +54 -0
- package/dist/types/feedback.d.ts +11 -0
- package/dist/types/files.d.ts +150 -0
- package/dist/types/index.d.ts +37 -0
- package/dist/types/infra.d.ts +188 -0
- package/dist/types/issues.d.ts +106 -0
- package/dist/types/jewels.d.ts +336 -0
- package/dist/types/methods.d.ts +34 -0
- package/dist/types/prerender.d.ts +59 -0
- package/dist/types/previews.d.ts +39 -0
- package/dist/types/progress.d.ts +31 -0
- package/dist/types/qaRecordings.d.ts +51 -0
- package/dist/types/releases.d.ts +233 -0
- package/dist/types/requests.d.ts +114 -0
- package/dist/types/secrets.d.ts +52 -0
- package/dist/types/settings.d.ts +79 -0
- package/dist/types/tasks.d.ts +99 -0
- package/dist/types/usage.d.ts +29 -0
- package/dist/types/users.d.ts +61 -0
- package/dist/types/voice.d.ts +197 -0
- package/dist/upload.d.ts +11 -0
- package/package.json +7 -10
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response types for warming, sampling and retrieval evals.
|
|
3
|
+
*
|
|
4
|
+
* Transcribed from the api (apps/api/src/):
|
|
5
|
+
* roles/http/routes/Apps/manage/dataSources/evals.ts (setView, runView)
|
|
6
|
+
* customerApps/AppDataSources/evals/types.ts
|
|
7
|
+
*/
|
|
8
|
+
import type { DataSourceJob } from './dataSourceJobs.js';
|
|
9
|
+
import type { OperationProgress } from './progress.js';
|
|
10
|
+
/**
|
|
11
|
+
* An index reload in flight, or its last failure: after an eviction on the
|
|
12
|
+
* shared pool, after a lost node or a resume on dedicated capacity, or a
|
|
13
|
+
* reindex. Null on the source when the index is simply resident.
|
|
14
|
+
*/
|
|
15
|
+
export interface DataSourcesHydration {
|
|
16
|
+
copied: number;
|
|
17
|
+
total: number;
|
|
18
|
+
startedAt: string;
|
|
19
|
+
error: string | null;
|
|
20
|
+
/** The reload in the one progress shape, with the platform's measured rate and ETA. */
|
|
21
|
+
progress: OperationProgress;
|
|
22
|
+
}
|
|
23
|
+
/** POST /datasources/hydrate */
|
|
24
|
+
export interface DataSourcesHydrateResult {
|
|
25
|
+
/** True when the points were already there and nothing was started. */
|
|
26
|
+
resident: boolean;
|
|
27
|
+
hydration: DataSourcesHydration | null;
|
|
28
|
+
}
|
|
29
|
+
/** POST /datasources/sample */
|
|
30
|
+
export interface DataSourcesSampleResult {
|
|
31
|
+
dataSource: {
|
|
32
|
+
id: string;
|
|
33
|
+
slug: string;
|
|
34
|
+
sampleOf: string;
|
|
35
|
+
pipelineVersion: number;
|
|
36
|
+
};
|
|
37
|
+
job: DataSourceJob;
|
|
38
|
+
}
|
|
39
|
+
/** `queued` until a worker claims the generation; hand-written sets start `ready`. */
|
|
40
|
+
export type EvalSetState = 'queued' | 'generating' | 'ready' | 'failed';
|
|
41
|
+
export type EvalQueryStyle = 'cloze' | 'question';
|
|
42
|
+
export interface EvalSet {
|
|
43
|
+
id: string;
|
|
44
|
+
dataSourceId: string;
|
|
45
|
+
name: string;
|
|
46
|
+
description: string | null;
|
|
47
|
+
state: EvalSetState;
|
|
48
|
+
generation: {
|
|
49
|
+
style: EvalQueryStyle;
|
|
50
|
+
size: number;
|
|
51
|
+
modelId: string | null;
|
|
52
|
+
seed: number;
|
|
53
|
+
tags: string[];
|
|
54
|
+
} | null;
|
|
55
|
+
queryCount: number;
|
|
56
|
+
/** Model spend generating questions (estimate); zero for cloze. */
|
|
57
|
+
estimatedCredits: number;
|
|
58
|
+
estimatedDollars: number;
|
|
59
|
+
error: string | null;
|
|
60
|
+
createdAt: string;
|
|
61
|
+
updatedAt: string;
|
|
62
|
+
}
|
|
63
|
+
/** What should come back for a query; the content hash is what crosses sources. */
|
|
64
|
+
export interface EvalExpectation {
|
|
65
|
+
contentHash: string;
|
|
66
|
+
documentId: string;
|
|
67
|
+
chunkIndex: number | null;
|
|
68
|
+
filename: string | null;
|
|
69
|
+
}
|
|
70
|
+
export interface EvalQuery {
|
|
71
|
+
id: string;
|
|
72
|
+
setId: string;
|
|
73
|
+
query: string;
|
|
74
|
+
expected: EvalExpectation[];
|
|
75
|
+
passage: string | null;
|
|
76
|
+
tags: string[];
|
|
77
|
+
origin: 'generated' | 'user';
|
|
78
|
+
createdAt: string;
|
|
79
|
+
}
|
|
80
|
+
/** Partial retrieval config plus mode, merged over the target's own. */
|
|
81
|
+
export interface EvalRetrieval {
|
|
82
|
+
mode?: 'hybrid' | 'semantic' | 'lexical';
|
|
83
|
+
hybrid?: boolean;
|
|
84
|
+
rerank?: {
|
|
85
|
+
enabled?: boolean;
|
|
86
|
+
modelId?: string | null;
|
|
87
|
+
candidates?: number;
|
|
88
|
+
};
|
|
89
|
+
adaptive?: {
|
|
90
|
+
enabled?: boolean;
|
|
91
|
+
gap?: number;
|
|
92
|
+
};
|
|
93
|
+
topK?: number;
|
|
94
|
+
}
|
|
95
|
+
export type EvalRunState = 'queued' | 'running' | 'done' | 'failed';
|
|
96
|
+
export interface EvalAggregates {
|
|
97
|
+
queries: number;
|
|
98
|
+
scored: number;
|
|
99
|
+
unresolvable: number;
|
|
100
|
+
errors: number;
|
|
101
|
+
recall: {
|
|
102
|
+
at1: number;
|
|
103
|
+
at5: number;
|
|
104
|
+
at10: number;
|
|
105
|
+
};
|
|
106
|
+
chunkRecall: {
|
|
107
|
+
at5: number;
|
|
108
|
+
at10: number;
|
|
109
|
+
queries: number;
|
|
110
|
+
} | null;
|
|
111
|
+
mrr: number;
|
|
112
|
+
ndcg10: number;
|
|
113
|
+
latency: {
|
|
114
|
+
p50: number;
|
|
115
|
+
p95: number;
|
|
116
|
+
mean: number;
|
|
117
|
+
};
|
|
118
|
+
estimatedCredits: number;
|
|
119
|
+
estimatedCreditsPerQuery: number;
|
|
120
|
+
rerankedShare: number;
|
|
121
|
+
/** Which branch found the expected document, over hits in the top 10. */
|
|
122
|
+
branches: {
|
|
123
|
+
denseOnly: number;
|
|
124
|
+
lexicalOnly: number;
|
|
125
|
+
both: number;
|
|
126
|
+
unexplained: number;
|
|
127
|
+
};
|
|
128
|
+
byTag: Record<string, {
|
|
129
|
+
queries: number;
|
|
130
|
+
recallAt5: number;
|
|
131
|
+
mrr: number;
|
|
132
|
+
}>;
|
|
133
|
+
truncatedTo: number | null;
|
|
134
|
+
computedAt: string;
|
|
135
|
+
}
|
|
136
|
+
export interface EvalRun {
|
|
137
|
+
id: string;
|
|
138
|
+
setId: string;
|
|
139
|
+
dataSourceId: string;
|
|
140
|
+
pipelineVersion: number;
|
|
141
|
+
label: string | null;
|
|
142
|
+
retrieval: EvalRetrieval;
|
|
143
|
+
state: EvalRunState;
|
|
144
|
+
completed: number;
|
|
145
|
+
total: number;
|
|
146
|
+
aggregates: EvalAggregates | null;
|
|
147
|
+
estimatedCredits: number;
|
|
148
|
+
estimatedDollars: number;
|
|
149
|
+
error: string | null;
|
|
150
|
+
createdAt: string;
|
|
151
|
+
startedAt: string | null;
|
|
152
|
+
finishedAt: string | null;
|
|
153
|
+
}
|
|
154
|
+
/** One row of a run's dataset. */
|
|
155
|
+
export interface EvalQueryResult {
|
|
156
|
+
queryId: string;
|
|
157
|
+
query: string;
|
|
158
|
+
tags: string[];
|
|
159
|
+
resolvable: boolean;
|
|
160
|
+
documentRank: number | null;
|
|
161
|
+
chunkRank: number | null;
|
|
162
|
+
matchedVia: 'dense' | 'lexical' | 'both' | null;
|
|
163
|
+
retrieved: {
|
|
164
|
+
documentId: string;
|
|
165
|
+
contentHash: string | null;
|
|
166
|
+
chunkIndex: number | null;
|
|
167
|
+
score: number;
|
|
168
|
+
retrievalRank: number | null;
|
|
169
|
+
matchedVia: 'dense' | 'lexical' | 'both' | null;
|
|
170
|
+
}[];
|
|
171
|
+
reranked: boolean;
|
|
172
|
+
latencyMs: number;
|
|
173
|
+
estimatedCredits: number;
|
|
174
|
+
error: string | null;
|
|
175
|
+
}
|
|
176
|
+
export interface EvalSetResult {
|
|
177
|
+
set: EvalSet;
|
|
178
|
+
}
|
|
179
|
+
export interface EvalSetsListResult {
|
|
180
|
+
sets: EvalSet[];
|
|
181
|
+
}
|
|
182
|
+
export interface EvalSetGetResult {
|
|
183
|
+
set: EvalSet;
|
|
184
|
+
dataSource: {
|
|
185
|
+
id: string;
|
|
186
|
+
slug: string;
|
|
187
|
+
} | null;
|
|
188
|
+
queries?: EvalQuery[];
|
|
189
|
+
}
|
|
190
|
+
export interface EvalQueriesAddResult {
|
|
191
|
+
added: number;
|
|
192
|
+
queryCount: number;
|
|
193
|
+
}
|
|
194
|
+
export interface EvalQueriesListResult {
|
|
195
|
+
queries: EvalQuery[];
|
|
196
|
+
}
|
|
197
|
+
export interface EvalRunResult {
|
|
198
|
+
run: EvalRun;
|
|
199
|
+
queries?: EvalQueryResult[];
|
|
200
|
+
}
|
|
201
|
+
export interface EvalRunsListResult {
|
|
202
|
+
runs: EvalRun[];
|
|
203
|
+
}
|
|
204
|
+
export interface EvalCompareResult {
|
|
205
|
+
a: EvalRun;
|
|
206
|
+
b: EvalRun;
|
|
207
|
+
/** Aggregate deltas, b minus a, for every numeric field. */
|
|
208
|
+
delta: Record<string, unknown>;
|
|
209
|
+
/** Per-query movement when both runs share a set; null otherwise. */
|
|
210
|
+
queries: {
|
|
211
|
+
wins: {
|
|
212
|
+
queryId: string;
|
|
213
|
+
query: string;
|
|
214
|
+
rankA: number | null;
|
|
215
|
+
rankB: number | null;
|
|
216
|
+
}[];
|
|
217
|
+
losses: {
|
|
218
|
+
queryId: string;
|
|
219
|
+
query: string;
|
|
220
|
+
rankA: number | null;
|
|
221
|
+
rankB: number | null;
|
|
222
|
+
}[];
|
|
223
|
+
unchanged: number;
|
|
224
|
+
} | null;
|
|
225
|
+
}
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response types for bulk ingestion jobs.
|
|
3
|
+
*
|
|
4
|
+
* Transcribed from the api (apps/api/src/):
|
|
5
|
+
* roles/http/routes/Apps/manage/dataSources/jobs.ts (jobView)
|
|
6
|
+
* customerApps/AppDataSources/jobs/types.ts
|
|
7
|
+
*/
|
|
8
|
+
import type { OperationProgress } from './progress.js';
|
|
9
|
+
/**
|
|
10
|
+
* Where a job's documents come from. `store` is every object under a prefix of
|
|
11
|
+
* one of the app's file stores (what `files put` fills); `manifest` is a JSONL
|
|
12
|
+
* object the CLI uploaded, one `{ url, filename?, externalId?, metadata? }` per
|
|
13
|
+
* line, for documents that live behind URLs; `connector` is a sync of the S3
|
|
14
|
+
* bucket the source is connected to (`datasources sync`); `remap` runs the
|
|
15
|
+
* live mapper over the source's own raw copies (`datasources remap`); `replay`
|
|
16
|
+
* runs an earlier job's quarantined objects through the current mapper
|
|
17
|
+
* (`datasources jobs replay`); `reread` reads the source's built documents
|
|
18
|
+
* again with its live reader (`datasources reread`).
|
|
19
|
+
*/
|
|
20
|
+
export type DataSourceJobSource = {
|
|
21
|
+
type: 'store';
|
|
22
|
+
store: string;
|
|
23
|
+
access: 'private' | 'public';
|
|
24
|
+
prefix: string;
|
|
25
|
+
} | {
|
|
26
|
+
type: 'manifest';
|
|
27
|
+
key: string;
|
|
28
|
+
} | {
|
|
29
|
+
type: 'connector';
|
|
30
|
+
connectorId: string;
|
|
31
|
+
} | {
|
|
32
|
+
/** A random draw of another source's documents (`datasources sample`). */
|
|
33
|
+
type: 'sample';
|
|
34
|
+
fromDataSourceId: string;
|
|
35
|
+
size: number;
|
|
36
|
+
filter: Record<string, unknown> | null;
|
|
37
|
+
stratify: string | null;
|
|
38
|
+
} | {
|
|
39
|
+
type: 'remap';
|
|
40
|
+
} | {
|
|
41
|
+
type: 'replay';
|
|
42
|
+
fromJobId: string;
|
|
43
|
+
kind: 'skip' | 'error' | null;
|
|
44
|
+
} | {
|
|
45
|
+
type: 'reread';
|
|
46
|
+
/** Only the documents the reader has not read. */
|
|
47
|
+
unread: boolean;
|
|
48
|
+
/** Only documents matching this metadata filter. */
|
|
49
|
+
filter: Record<string, unknown> | null;
|
|
50
|
+
} | {
|
|
51
|
+
/**
|
|
52
|
+
* The rest of one file a `datasources add` (or an app's `add()`) took
|
|
53
|
+
* that maps to more than a thousand documents; `ref` is its filename.
|
|
54
|
+
*/
|
|
55
|
+
type: 'object';
|
|
56
|
+
ref: string;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* planning → planned → running ⇄ paused → done | failed | cancelled.
|
|
60
|
+
* `planned` waits for `approve` (or was started with `approve: true` and a
|
|
61
|
+
* gate refused; the job's `error` says which).
|
|
62
|
+
*/
|
|
63
|
+
export type DataSourceJobState = 'planning' | 'planned' | 'running' | 'paused' | 'done' | 'failed' | 'cancelled';
|
|
64
|
+
/**
|
|
65
|
+
* What the corpus would cost before any of it is spent. Every number is an
|
|
66
|
+
* extrapolation from a sample of about a hundred objects; credits are ledger
|
|
67
|
+
* credits (nano-dollars) at today's catalog rates.
|
|
68
|
+
*/
|
|
69
|
+
export interface DataSourceJobPlan {
|
|
70
|
+
objects: number;
|
|
71
|
+
bytes: number;
|
|
72
|
+
sample: {
|
|
73
|
+
documents: number;
|
|
74
|
+
failed: number;
|
|
75
|
+
avgChunks: number;
|
|
76
|
+
avgTokens: number;
|
|
77
|
+
avgPages: number;
|
|
78
|
+
modelExtractionShare: number;
|
|
79
|
+
secondsPerDocument: number;
|
|
80
|
+
};
|
|
81
|
+
/**
|
|
82
|
+
* Mapped sources only: what the mapper did with the sampled objects. The
|
|
83
|
+
* skip share is the baseline the run's skip gate pauses against.
|
|
84
|
+
*/
|
|
85
|
+
mapping: {
|
|
86
|
+
objects: number;
|
|
87
|
+
documentsPerObject: number;
|
|
88
|
+
passthroughShare: number;
|
|
89
|
+
skipShare: number;
|
|
90
|
+
errorShare: number;
|
|
91
|
+
deleteShare: number;
|
|
92
|
+
secondsPerObject: number;
|
|
93
|
+
/** What the mapper's own model and step calls billed per sampled object. */
|
|
94
|
+
creditsPerObject?: number;
|
|
95
|
+
/** What the sample's mapper calls billed, spent before the run began. */
|
|
96
|
+
sampleCredits?: number;
|
|
97
|
+
} | null;
|
|
98
|
+
/**
|
|
99
|
+
* Sources with a live reader: what reading costs. `history` prices it from
|
|
100
|
+
* what the live reader has done (a hundred documents or more); `slice`
|
|
101
|
+
* from a reread's first slice, read for real before the plan (its reads
|
|
102
|
+
* are done, and the run leaves them out); null when unmeasured, and the
|
|
103
|
+
* budget alone bounds the reader. The failure share is the baseline the
|
|
104
|
+
* run's reader gate pauses against.
|
|
105
|
+
*/
|
|
106
|
+
reading?: {
|
|
107
|
+
measured: 'history' | 'slice' | null;
|
|
108
|
+
creditsPerDocument: number | null;
|
|
109
|
+
sampleCredits?: number;
|
|
110
|
+
sampleDocuments?: number;
|
|
111
|
+
sampleRead?: number;
|
|
112
|
+
/** How much the live database grew per document the slice read. */
|
|
113
|
+
bytesPerDocument?: number | null;
|
|
114
|
+
baselineFailureShare?: number;
|
|
115
|
+
} | null;
|
|
116
|
+
projected: {
|
|
117
|
+
documents: number;
|
|
118
|
+
chunks: number;
|
|
119
|
+
tokens: number;
|
|
120
|
+
points: number;
|
|
121
|
+
artifactBytes: number;
|
|
122
|
+
credits: {
|
|
123
|
+
embedding: number;
|
|
124
|
+
extraction: number;
|
|
125
|
+
contextual: number;
|
|
126
|
+
/** Mapped sources: the mapper's own calls over every object, the sample's included. */
|
|
127
|
+
mapping?: number;
|
|
128
|
+
/** Reader sources: the reader's own calls over every document, a slice's included. */
|
|
129
|
+
reading?: number;
|
|
130
|
+
total: number;
|
|
131
|
+
};
|
|
132
|
+
/** A reread: the live database's size once every document is read. */
|
|
133
|
+
databaseBytes?: number | null;
|
|
134
|
+
};
|
|
135
|
+
capacity: {
|
|
136
|
+
placement: 'shared' | 'dedicated';
|
|
137
|
+
resourceId: string | null;
|
|
138
|
+
/**
|
|
139
|
+
* Chunks the placement holds in total at this source's dimensions: the
|
|
140
|
+
* shared pool's per-source cap, or what the resource would hold were this
|
|
141
|
+
* source alone on it.
|
|
142
|
+
*/
|
|
143
|
+
maxPoints: number;
|
|
144
|
+
/** Chunks already there (other sources on the resource, or this one's own). */
|
|
145
|
+
heldPoints: number;
|
|
146
|
+
/** False means approve is refused until the source moves or the resource grows. */
|
|
147
|
+
fits: boolean;
|
|
148
|
+
/**
|
|
149
|
+
* Dedicated only: the bytes the decision was made on. `binding` names the
|
|
150
|
+
* limit that refused; `held`, `incoming` and `usable` are disk and RAM.
|
|
151
|
+
*/
|
|
152
|
+
footprint?: {
|
|
153
|
+
dimensions: number;
|
|
154
|
+
fits: boolean;
|
|
155
|
+
binding: 'disk' | 'ram' | null;
|
|
156
|
+
held: {
|
|
157
|
+
diskBytes: number;
|
|
158
|
+
ramBytes: number;
|
|
159
|
+
};
|
|
160
|
+
incoming: {
|
|
161
|
+
diskBytes: number;
|
|
162
|
+
ramBytes: number;
|
|
163
|
+
};
|
|
164
|
+
usable: {
|
|
165
|
+
diskBytes: number;
|
|
166
|
+
ramBytes: number;
|
|
167
|
+
};
|
|
168
|
+
} | null;
|
|
169
|
+
};
|
|
170
|
+
durationMinutes: number;
|
|
171
|
+
warnings: string[];
|
|
172
|
+
computedAt: string;
|
|
173
|
+
}
|
|
174
|
+
export interface DataSourceJob {
|
|
175
|
+
id: string;
|
|
176
|
+
dataSourceId: string;
|
|
177
|
+
pipelineId: string;
|
|
178
|
+
source: DataSourceJobSource;
|
|
179
|
+
/**
|
|
180
|
+
* Objects the mapper skipped or failed on, and documents the reader failed
|
|
181
|
+
* on. Counted on `jobs status`; null on the list.
|
|
182
|
+
*/
|
|
183
|
+
quarantine: {
|
|
184
|
+
skipped: number;
|
|
185
|
+
errors: number;
|
|
186
|
+
readErrors: number;
|
|
187
|
+
} | null;
|
|
188
|
+
state: DataSourceJobState;
|
|
189
|
+
autoApprove: boolean;
|
|
190
|
+
plan: DataSourceJobPlan | null;
|
|
191
|
+
budgetCredits: number | null;
|
|
192
|
+
budgetDollars: number | null;
|
|
193
|
+
maxInflightBatches: number;
|
|
194
|
+
/** Embedding at the provider's priority tier (1.5x the embedding price). */
|
|
195
|
+
priority: boolean;
|
|
196
|
+
limitObjects: number | null;
|
|
197
|
+
enumerationDone: boolean;
|
|
198
|
+
counts: {
|
|
199
|
+
/** Keys the planning walk listed, before the change diff: progress, not work. */
|
|
200
|
+
objectsListed: number;
|
|
201
|
+
objectsSeen: number;
|
|
202
|
+
bytesSeen: number;
|
|
203
|
+
objectsDispatched: number;
|
|
204
|
+
documentsDone: number;
|
|
205
|
+
documentsSkipped: number;
|
|
206
|
+
documentsFailed: number;
|
|
207
|
+
chunks: number;
|
|
208
|
+
embedTokens: number;
|
|
209
|
+
extractionPages: number;
|
|
210
|
+
};
|
|
211
|
+
/**
|
|
212
|
+
* Spend so far: embedding, extraction and contextual at catalog rates, plus
|
|
213
|
+
* what the mapper's own calls billed. An estimate, not the ledger. The
|
|
214
|
+
* budget pauses the job on it.
|
|
215
|
+
*/
|
|
216
|
+
estimatedCredits: number;
|
|
217
|
+
estimatedDollars: number;
|
|
218
|
+
/**
|
|
219
|
+
* What frames billed for their own calls: the mapper's and the reader's,
|
|
220
|
+
* the plan's sample included.
|
|
221
|
+
*/
|
|
222
|
+
mapperCredits?: number;
|
|
223
|
+
mapperDollars?: number;
|
|
224
|
+
/** A running job's read stage on a reader source: documents read of those cut. */
|
|
225
|
+
reading?: {
|
|
226
|
+
documentsRead: number;
|
|
227
|
+
documentsTotal: number;
|
|
228
|
+
} | null;
|
|
229
|
+
/** Recent per-document failures, newest first, at most twenty. */
|
|
230
|
+
samples: {
|
|
231
|
+
ref: string;
|
|
232
|
+
message: string;
|
|
233
|
+
at: string;
|
|
234
|
+
}[];
|
|
235
|
+
error: string | null;
|
|
236
|
+
/**
|
|
237
|
+
* Why a running job stopped: the owner, the budget, repeated batch
|
|
238
|
+
* failures, (mapped sources) a skip share far above the plan's, or (reader
|
|
239
|
+
* sources) a reader failing on far more documents than it did at first.
|
|
240
|
+
*/
|
|
241
|
+
pauseReason: 'user' | 'budget' | 'errors' | 'skips' | 'reader' | null;
|
|
242
|
+
stalled: boolean;
|
|
243
|
+
/** Times a pause for errors resumed itself on its own schedule. */
|
|
244
|
+
autoResumes: number;
|
|
245
|
+
/** When a pause for errors will next resume itself; null when it waits for you. */
|
|
246
|
+
autoResumeAt: string | null;
|
|
247
|
+
createdAt: string;
|
|
248
|
+
startedAt: string | null;
|
|
249
|
+
finishedAt: string | null;
|
|
250
|
+
lastProgressAt: string | null;
|
|
251
|
+
/**
|
|
252
|
+
* The job in the one progress shape: objects listed while planning,
|
|
253
|
+
* documents processed against the plan's projection after, with the
|
|
254
|
+
* coordinator's measured rate and ETA.
|
|
255
|
+
*/
|
|
256
|
+
progress: OperationProgress;
|
|
257
|
+
}
|
|
258
|
+
/** GET /datasources/jobs */
|
|
259
|
+
export interface DataSourceJobsListResult {
|
|
260
|
+
jobs: DataSourceJob[];
|
|
261
|
+
}
|
|
262
|
+
/** GET /datasources/jobs/:id */
|
|
263
|
+
export interface DataSourceJobGetResult {
|
|
264
|
+
job: DataSourceJob;
|
|
265
|
+
dataSource: {
|
|
266
|
+
id: string;
|
|
267
|
+
slug: string;
|
|
268
|
+
};
|
|
269
|
+
batches: Record<'pending' | 'queued' | 'running' | 'done' | 'failed' | 'cancelled', number>;
|
|
270
|
+
}
|
|
271
|
+
/** POST /datasources/jobs and the four controls */
|
|
272
|
+
export interface DataSourceJobResult {
|
|
273
|
+
job: DataSourceJob;
|
|
274
|
+
}
|
|
275
|
+
/** POST /datasources/jobs/manifest-token */
|
|
276
|
+
export interface DataSourceJobManifestTokenResult {
|
|
277
|
+
key: string;
|
|
278
|
+
upload: {
|
|
279
|
+
key: string;
|
|
280
|
+
uploadUrl: string;
|
|
281
|
+
uploadFields: Record<string, string>;
|
|
282
|
+
};
|
|
283
|
+
}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Response types for data-source mappers: inspecting raw objects, testing a
|
|
3
|
+
* mapper, remapping, and a job's quarantine.
|
|
4
|
+
*
|
|
5
|
+
* Transcribed from the api (apps/api/src/):
|
|
6
|
+
* roles/http/routes/Apps/manage/dataSources/mappers.ts
|
|
7
|
+
* customerApps/AppDataSources/mapping.ts (mapperView, buildInspectReport)
|
|
8
|
+
*/
|
|
9
|
+
import type { DataSourceJob } from './dataSourceJobs.js';
|
|
10
|
+
/**
|
|
11
|
+
* The mapper a source runs: the file it was compiled from, its version, and
|
|
12
|
+
* the build it came from. The build fields are null for a tunnel run of local
|
|
13
|
+
* source (`map test --dev`).
|
|
14
|
+
*/
|
|
15
|
+
export interface DataSourceMapper {
|
|
16
|
+
path: string;
|
|
17
|
+
/** The compiled bundle key, or `dev:<sessionId>` for a tunnel run; stamped on the documents it produces. */
|
|
18
|
+
mapperKey: string;
|
|
19
|
+
timeoutMs: number;
|
|
20
|
+
releaseId: string | null;
|
|
21
|
+
branch: string | null;
|
|
22
|
+
commitSha: string | null;
|
|
23
|
+
/** When this bundle became the source's active mapper. */
|
|
24
|
+
activatedAt: string | null;
|
|
25
|
+
}
|
|
26
|
+
/** POST /datasources/mapper/deploy */
|
|
27
|
+
export interface DataSourceMapDeployResult {
|
|
28
|
+
dataSource: {
|
|
29
|
+
id: string;
|
|
30
|
+
slug: string;
|
|
31
|
+
};
|
|
32
|
+
mapper: DataSourceMapper;
|
|
33
|
+
/** False when the source already ran this bundle. */
|
|
34
|
+
changed: boolean;
|
|
35
|
+
}
|
|
36
|
+
/** Which objects an inspect or map test looked at. */
|
|
37
|
+
export type DataSourceObjectSelection = {
|
|
38
|
+
kind: 'store';
|
|
39
|
+
store: string;
|
|
40
|
+
access: 'private' | 'public';
|
|
41
|
+
prefix: string;
|
|
42
|
+
} | {
|
|
43
|
+
kind: 'connector';
|
|
44
|
+
};
|
|
45
|
+
export interface DataSourceInspectReport {
|
|
46
|
+
objects: number;
|
|
47
|
+
bytes: number;
|
|
48
|
+
/** True when the listing stopped at the limit. */
|
|
49
|
+
truncated: boolean;
|
|
50
|
+
byExtension: {
|
|
51
|
+
extension: string;
|
|
52
|
+
count: number;
|
|
53
|
+
bytes: number;
|
|
54
|
+
}[];
|
|
55
|
+
/** Keys grouped by shape, digit runs and ids collapsed: `2024/#/{uuid}.json`. */
|
|
56
|
+
clusters: {
|
|
57
|
+
pattern: string;
|
|
58
|
+
extension: string;
|
|
59
|
+
count: number;
|
|
60
|
+
bytes: number;
|
|
61
|
+
examples: string[];
|
|
62
|
+
}[];
|
|
63
|
+
sizeBuckets: {
|
|
64
|
+
label: string;
|
|
65
|
+
count: number;
|
|
66
|
+
}[];
|
|
67
|
+
/** Up to ten objects read (first 256 KB each). */
|
|
68
|
+
samples: {
|
|
69
|
+
key: string;
|
|
70
|
+
size: number | null;
|
|
71
|
+
contentType: string | null;
|
|
72
|
+
encoding: 'utf-8' | 'binary';
|
|
73
|
+
empty: boolean;
|
|
74
|
+
/** Top-level keys when the head parsed as a JSON object. */
|
|
75
|
+
jsonKeys: string[] | null;
|
|
76
|
+
/** Line count for text that was not one JSON object (JSONL, CSV, ...). */
|
|
77
|
+
lines: number | null;
|
|
78
|
+
textLength: number;
|
|
79
|
+
head: string;
|
|
80
|
+
}[];
|
|
81
|
+
/** Distinct top-level JSON key sets across the samples, most common first. */
|
|
82
|
+
jsonSignatures: {
|
|
83
|
+
keys: string[];
|
|
84
|
+
count: number;
|
|
85
|
+
}[];
|
|
86
|
+
}
|
|
87
|
+
/** POST /datasources/inspect */
|
|
88
|
+
export interface DataSourceInspectResult {
|
|
89
|
+
selection: DataSourceObjectSelection;
|
|
90
|
+
report: DataSourceInspectReport;
|
|
91
|
+
}
|
|
92
|
+
/** One mapped document as a map test previews it (markdown truncated). */
|
|
93
|
+
export interface DataSourceMapTestDocument {
|
|
94
|
+
externalId: string;
|
|
95
|
+
title: string;
|
|
96
|
+
metadata: Record<string, string | number | boolean> | null;
|
|
97
|
+
replaces: string[];
|
|
98
|
+
markdownBytes: number;
|
|
99
|
+
preview: string;
|
|
100
|
+
}
|
|
101
|
+
export type DataSourceMapTestOutcome = {
|
|
102
|
+
kind: 'documents';
|
|
103
|
+
/** The first twenty of them. */
|
|
104
|
+
documents: DataSourceMapTestDocument[];
|
|
105
|
+
/** Every valid document the object produced. */
|
|
106
|
+
count: number;
|
|
107
|
+
/**
|
|
108
|
+
* Its documents that failed validation, by their index in the mapper's
|
|
109
|
+
* output: a job leaves them out and quarantines the object for them.
|
|
110
|
+
* Null when every one passed.
|
|
111
|
+
*/
|
|
112
|
+
invalid: {
|
|
113
|
+
count: number;
|
|
114
|
+
examples: {
|
|
115
|
+
document: number;
|
|
116
|
+
reason: string;
|
|
117
|
+
}[];
|
|
118
|
+
} | null;
|
|
119
|
+
/** Parts of the frame's documents that storage did not have. */
|
|
120
|
+
missingParts?: number;
|
|
121
|
+
} | {
|
|
122
|
+
kind: 'passthrough';
|
|
123
|
+
metadata: Record<string, string | number | boolean> | null;
|
|
124
|
+
} | {
|
|
125
|
+
kind: 'skip';
|
|
126
|
+
reason: string;
|
|
127
|
+
} | {
|
|
128
|
+
kind: 'deletes';
|
|
129
|
+
externalIds: string[];
|
|
130
|
+
} | {
|
|
131
|
+
kind: 'error';
|
|
132
|
+
message: string;
|
|
133
|
+
stack?: string;
|
|
134
|
+
};
|
|
135
|
+
/** POST /datasources/map-test */
|
|
136
|
+
export interface DataSourceMapTestResult {
|
|
137
|
+
mapper: DataSourceMapper;
|
|
138
|
+
/** The mapper ran from local source through the dev session. */
|
|
139
|
+
dev: boolean;
|
|
140
|
+
objects: number;
|
|
141
|
+
/** Documents produced across every `documents` outcome. */
|
|
142
|
+
produced: number;
|
|
143
|
+
counts: {
|
|
144
|
+
documents: number;
|
|
145
|
+
passthrough: number;
|
|
146
|
+
skip: number;
|
|
147
|
+
deletes: number;
|
|
148
|
+
error: number;
|
|
149
|
+
};
|
|
150
|
+
results: {
|
|
151
|
+
key: string;
|
|
152
|
+
outcome: DataSourceMapTestOutcome;
|
|
153
|
+
durationMs: number;
|
|
154
|
+
}[];
|
|
155
|
+
}
|
|
156
|
+
/** POST /datasources/remap, POST /datasources/jobs/:id/replay */
|
|
157
|
+
export interface DataSourceRemapResult {
|
|
158
|
+
job: DataSourceJob;
|
|
159
|
+
mapper: DataSourceMapper;
|
|
160
|
+
dataSource: {
|
|
161
|
+
id: string;
|
|
162
|
+
slug: string;
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* One object a job's mapper did not turn into documents (`stage: 'map'`), or
|
|
167
|
+
* one document its reader failed on (`stage: 'read'`).
|
|
168
|
+
*/
|
|
169
|
+
export interface DataSourceQuarantinedObject {
|
|
170
|
+
id: string;
|
|
171
|
+
/** The key or URL the object was known by; a document id for a read. */
|
|
172
|
+
ref: string;
|
|
173
|
+
kind: 'skip' | 'error';
|
|
174
|
+
stage: 'map' | 'read';
|
|
175
|
+
reason: string;
|
|
176
|
+
/** False when the object's raw copy never landed — a replay cannot reach it. */
|
|
177
|
+
replayable: boolean;
|
|
178
|
+
at: string;
|
|
179
|
+
}
|
|
180
|
+
/** GET /datasources/jobs/:id/quarantine */
|
|
181
|
+
export interface DataSourceQuarantineResult {
|
|
182
|
+
job: {
|
|
183
|
+
id: string;
|
|
184
|
+
state: string;
|
|
185
|
+
};
|
|
186
|
+
dataSource: {
|
|
187
|
+
id: string;
|
|
188
|
+
slug: string;
|
|
189
|
+
};
|
|
190
|
+
counts: {
|
|
191
|
+
skipped: number;
|
|
192
|
+
errors: number;
|
|
193
|
+
readErrors: number;
|
|
194
|
+
};
|
|
195
|
+
/** Reasons by frequency, most common first. */
|
|
196
|
+
reasons: {
|
|
197
|
+
kind: 'skip' | 'error';
|
|
198
|
+
stage: 'map' | 'read';
|
|
199
|
+
reason: string;
|
|
200
|
+
count: number;
|
|
201
|
+
}[];
|
|
202
|
+
objects: DataSourceQuarantinedObject[];
|
|
203
|
+
nextCursor: string | null;
|
|
204
|
+
}
|