@frontera-sdk/cli 1.45.7 → 1.45.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/blueprint-authoring-api.ts +20 -2
- package/src/api/dataset-api.ts +34 -1
- package/src/api/validation-detail.ts +87 -0
- package/src/commands/agent/index-commands.ts +24 -7
- package/src/commands/blueprint/authoring.ts +37 -7
- package/src/commands/blueprint/declarative.ts +7 -1
- package/src/commands/blueprint/grants.ts +32 -5
- package/src/commands/dataset/index-commands.ts +152 -5
- package/src/commands/knowledge/index-commands.ts +21 -2
- package/src/commands/knowledge/upload-batch.ts +55 -0
- package/src/commands/source/index-commands.ts +79 -1
- package/src/vendor/kit-assets.json +9 -9
- package/src/vendor/sdk-sources.json +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frontera-sdk/cli",
|
|
3
|
-
"version": "1.45.
|
|
3
|
+
"version": "1.45.8",
|
|
4
4
|
"description": "The frontera CLI — scaffold, pull, save and deploy Frontera apps and automations.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"frontera",
|
|
@@ -38,13 +38,13 @@
|
|
|
38
38
|
"build:release": "bun run scripts/build-release.ts"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@frontera-sdk/automation": "1.45.
|
|
42
|
-
"@frontera-sdk/core": "1.45.
|
|
41
|
+
"@frontera-sdk/automation": "1.45.7",
|
|
42
|
+
"@frontera-sdk/core": "1.45.7",
|
|
43
43
|
"gray-matter": "^4.0.3",
|
|
44
44
|
"yaml": "^2.9.0"
|
|
45
45
|
},
|
|
46
46
|
"devDependencies": {
|
|
47
|
-
"@types/bun": "
|
|
47
|
+
"@types/bun": "^1.3.14",
|
|
48
48
|
"typescript": "^5.9.3"
|
|
49
49
|
}
|
|
50
50
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CliError } from '../errors'
|
|
2
2
|
import { credentialFailure } from './credential-failure'
|
|
3
|
+
import { formatValidationDetails } from './validation-detail'
|
|
3
4
|
import type { DefinitionBundle } from '../blueprint/model'
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -60,7 +61,16 @@ export class BlueprintAuthoringApi {
|
|
|
60
61
|
hint: 'frontera blueprint status',
|
|
61
62
|
})
|
|
62
63
|
}
|
|
63
|
-
|
|
64
|
+
// The service's refusal often carries a TypeBox details array naming the
|
|
65
|
+
// exact fields — `/pluralName`, a valueType outside the enum — and this
|
|
66
|
+
// path used to flatten all of it to "Validation failed". On the verb that
|
|
67
|
+
// WRITES the draft, that reasonless refusal was the whole error a caller
|
|
68
|
+
// saw, and the audit read it as an unusable write path.
|
|
69
|
+
const detail = formatValidationDetails(payload?.details)
|
|
70
|
+
throw new CliError(payload?.message ?? `${res.status} ${text.slice(0, 200)}`, {
|
|
71
|
+
code: (payload?.code as string) ?? 'FAILURE',
|
|
72
|
+
hint: detail ?? 'frontera blueprint status',
|
|
73
|
+
})
|
|
64
74
|
}
|
|
65
75
|
|
|
66
76
|
return (payload?.data ?? payload) as T
|
|
@@ -313,7 +323,15 @@ export class BlueprintAuthoringApi {
|
|
|
313
323
|
|
|
314
324
|
// ── release lifecycle ─────────────────────────────────────────────────────
|
|
315
325
|
|
|
316
|
-
validate(): Promise<{
|
|
326
|
+
validate(): Promise<{
|
|
327
|
+
id?: string
|
|
328
|
+
reportId?: string
|
|
329
|
+
draftRevision?: number
|
|
330
|
+
draftDigest?: string
|
|
331
|
+
baseReleaseDigest?: string
|
|
332
|
+
catalogProjection?: { objectTypes?: unknown[]; linkTypes?: unknown[] }
|
|
333
|
+
evolution?: { changes?: unknown[] }
|
|
334
|
+
}> {
|
|
317
335
|
return this.call('/v1/blueprint/lifecycle/validate', { method: 'POST', body: {} })
|
|
318
336
|
}
|
|
319
337
|
|
package/src/api/dataset-api.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { basename } from 'node:path'
|
|
|
3
3
|
|
|
4
4
|
import { CliError } from '../errors'
|
|
5
5
|
import { credentialFailure } from './credential-failure'
|
|
6
|
+
import { formatValidationDetails } from './validation-detail'
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* Datasets, from the CLI.
|
|
@@ -105,9 +106,19 @@ export class DatasetApi {
|
|
|
105
106
|
// bare "Insufficient permissions" and no route out.
|
|
106
107
|
const credential = credentialFailure(response.status, payload?.message, { token: this.token })
|
|
107
108
|
if (credential) throw credential
|
|
109
|
+
// `details` is TypeBox's own error array. Stringified it was unreadable,
|
|
110
|
+
// and this is the last gate before a Source exists.
|
|
111
|
+
const detail = formatValidationDetails(payload?.details)
|
|
108
112
|
throw new CliError(payload?.message ?? `Request failed (${response.status}).`, {
|
|
109
113
|
code: payload?.code ?? 'FAILURE',
|
|
110
|
-
|
|
114
|
+
// A refused CONNECTION is not a malformed document, and telling someone
|
|
115
|
+
// to re-read their file sends them to the wrong place entirely: the
|
|
116
|
+
// service dials the database itself, from wherever it runs.
|
|
117
|
+
hint: detail
|
|
118
|
+
?? (response.status === 409
|
|
119
|
+
? 'the service dials the database itself — check host and port are reachable '
|
|
120
|
+
+ 'from where the service runs, and that the password on stdin is current'
|
|
121
|
+
: 'check the document against `frontera source create --help`'),
|
|
111
122
|
})
|
|
112
123
|
}
|
|
113
124
|
return (payload?.data ?? payload) as T
|
|
@@ -288,6 +299,28 @@ export class DatasetApi {
|
|
|
288
299
|
return this.call<DatasetSummary>('/v1/data-integration/datasets/from-file', { form })
|
|
289
300
|
}
|
|
290
301
|
|
|
302
|
+
/**
|
|
303
|
+
* Rows into a dataset that already exists, as a new snapshot.
|
|
304
|
+
*
|
|
305
|
+
* `createFromFile` is CREATE-time only. A blank dataset — declared columns,
|
|
306
|
+
* no rows, and the common first step — had no way to receive data afterwards
|
|
307
|
+
* through the CLI, so an agent told its user the capability did not exist.
|
|
308
|
+
* It does: the service republishes against the CURRENT column contract and
|
|
309
|
+
* repoints the dataset atomically, keeping the prior snapshot on any failure.
|
|
310
|
+
*
|
|
311
|
+
* The file must match that contract exactly — names, types, nullability. A
|
|
312
|
+
* schema change is refused here by design; it is a new revision, not a
|
|
313
|
+
* re-upload.
|
|
314
|
+
*/
|
|
315
|
+
uploadSnapshot(datasetId: string, path: string): Promise<DatasetSummary> {
|
|
316
|
+
const form = new FormData()
|
|
317
|
+
form.append('file', new Blob([readFileSync(path)]), basename(path))
|
|
318
|
+
return this.call<DatasetSummary>(
|
|
319
|
+
`/v1/data-integration/datasets/${encodeURIComponent(datasetId)}/snapshots`,
|
|
320
|
+
{ form },
|
|
321
|
+
)
|
|
322
|
+
}
|
|
323
|
+
|
|
291
324
|
createBlank(payload: {
|
|
292
325
|
apiName: string
|
|
293
326
|
displayName: string
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turn the service's validation `details` into something a person can act on.
|
|
3
|
+
*
|
|
4
|
+
* The service answers a rejected document with TypeBox's own error array, and
|
|
5
|
+
* the CLI put it on screen with `JSON.stringify`:
|
|
6
|
+
*
|
|
7
|
+
* [{"summary":"Expected 'virtual'","type":32,"schema":{"const":"virtual",
|
|
8
|
+
* "type":"string"},"path":"/datasets/0/mode","message":"Expected 'virtual'",
|
|
9
|
+
* "errors":[]}]
|
|
10
|
+
*
|
|
11
|
+
* Every fact a caller needs is in there, and none of it is legible. The path is
|
|
12
|
+
* buried mid-line, the schema fragment is noise, and `type: 32` is an internal
|
|
13
|
+
* enum. `source create` was unusable through this: it is the last gate before a
|
|
14
|
+
* Source exists, and it answered with a machine's notes.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
interface TypeBoxError {
|
|
18
|
+
path?: string
|
|
19
|
+
message?: string
|
|
20
|
+
summary?: string
|
|
21
|
+
value?: unknown
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** `/datasets/0/mode` reads better as `datasets[0].mode`. */
|
|
25
|
+
export function readablePath(path: string | undefined): string {
|
|
26
|
+
if (!path || path === '/') return '(document root)'
|
|
27
|
+
return path
|
|
28
|
+
.replace(/^\//, '')
|
|
29
|
+
.split('/')
|
|
30
|
+
.map((segment) => (/^\d+$/.test(segment) ? `[${segment}]` : `.${segment}`))
|
|
31
|
+
.join('')
|
|
32
|
+
.replace(/^\./, '')
|
|
33
|
+
.replace(/\.\[/g, '[')
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Whether these errors all point at a discriminator.
|
|
38
|
+
*
|
|
39
|
+
* A discriminated union reports the field it switches on when ANY branch fails,
|
|
40
|
+
* so `mode: "virtual"` is rejected with "Expected 'virtual'" while the real
|
|
41
|
+
* defect is somewhere else in that branch entirely. Silently correct and
|
|
42
|
+
* completely misleading, and it cost an audit an afternoon — so when the shape
|
|
43
|
+
* matches, the hint says so rather than letting the caller stare at a field
|
|
44
|
+
* that is already right.
|
|
45
|
+
*/
|
|
46
|
+
export function looksLikeDiscriminator(errors: TypeBoxError[]): boolean {
|
|
47
|
+
return (
|
|
48
|
+
errors.length > 0
|
|
49
|
+
&& errors.every((e) => /^Expected '.*'$/.test(e.message ?? e.summary ?? ''))
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* One line per error, plus a note when the paths cannot be trusted.
|
|
55
|
+
*
|
|
56
|
+
* Returns `undefined` when there is nothing structured to say, so the caller
|
|
57
|
+
* keeps whatever hint it already had rather than showing an empty one.
|
|
58
|
+
*/
|
|
59
|
+
export function formatValidationDetails(details: unknown): string | undefined {
|
|
60
|
+
const errors = Array.isArray(details) ? (details as TypeBoxError[]) : null
|
|
61
|
+
if (!errors || errors.length === 0) return undefined
|
|
62
|
+
|
|
63
|
+
const lines = errors
|
|
64
|
+
.filter((e) => e && typeof e === 'object')
|
|
65
|
+
.map((e) => {
|
|
66
|
+
const message = e.message ?? e.summary ?? 'is not valid'
|
|
67
|
+
return `${readablePath(e.path)} — ${message}`
|
|
68
|
+
})
|
|
69
|
+
if (lines.length === 0) return undefined
|
|
70
|
+
|
|
71
|
+
// Capped: a document with fifty problems produces fifty lines, and the first
|
|
72
|
+
// few are the ones anybody reads. The count says what was left out rather
|
|
73
|
+
// than letting the list end without saying so.
|
|
74
|
+
const SHOWN = 5
|
|
75
|
+
const shown = lines.slice(0, SHOWN)
|
|
76
|
+
if (lines.length > SHOWN) shown.push(`… and ${lines.length - SHOWN} more`)
|
|
77
|
+
|
|
78
|
+
if (looksLikeDiscriminator(errors)) {
|
|
79
|
+
shown.push(
|
|
80
|
+
'that field may already be correct — a discriminated union reports the '
|
|
81
|
+
+ 'field it switches on when any part of that branch fails, so look at '
|
|
82
|
+
+ 'the rest of the same object',
|
|
83
|
+
)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return shown.join('\n ')
|
|
87
|
+
}
|
|
@@ -368,13 +368,26 @@ const diff: Command = {
|
|
|
368
368
|
},
|
|
369
369
|
async run(ctx) {
|
|
370
370
|
const t = await target(ctx, 'diff')
|
|
371
|
-
|
|
371
|
+
// Both, before either branch answers: "the live version is current" is as
|
|
372
|
+
// wrong for an agent that has never published as "matches the live
|
|
373
|
+
// version" is, and that branch used to answer without ever looking.
|
|
374
|
+
const [draft, live] = await Promise.all([
|
|
375
|
+
t.api.agentDraft(t.id),
|
|
376
|
+
t.api.agent(t.id) as Promise<Record<string, unknown>>,
|
|
377
|
+
])
|
|
378
|
+
const published = Boolean(live.currentVersion ?? live.currentVersionId)
|
|
372
379
|
|
|
373
380
|
if (!draft) {
|
|
374
|
-
return {
|
|
381
|
+
return {
|
|
382
|
+
data: { hasDraft: false, published, changed: [] },
|
|
383
|
+
text: published
|
|
384
|
+
? 'No draft staged — the live version is current.'
|
|
385
|
+
: 'No draft staged, and nothing is published — this agent has no configuration '
|
|
386
|
+
+ 'of its own yet.\n'
|
|
387
|
+
+ ` Stage one with \`frontera agent apply ${t.ref} --file <path>\`.`,
|
|
388
|
+
}
|
|
375
389
|
}
|
|
376
390
|
|
|
377
|
-
const live = (await t.api.agent(t.id)) as Record<string, unknown>
|
|
378
391
|
const liveConfig = (live.config ?? live) as Record<string, unknown>
|
|
379
392
|
const draftConfig = (draft.snapshot.config ?? {}) as Record<string, unknown>
|
|
380
393
|
|
|
@@ -421,11 +434,15 @@ const diff: Command = {
|
|
|
421
434
|
].sort()
|
|
422
435
|
|
|
423
436
|
return {
|
|
424
|
-
data: { hasDraft: true, revision: draft.revision, changed },
|
|
437
|
+
data: { hasDraft: true, revision: draft.revision, published, changed },
|
|
425
438
|
text:
|
|
426
|
-
changed.length
|
|
427
|
-
? `Draft revision ${draft.revision}
|
|
428
|
-
:
|
|
439
|
+
changed.length > 0
|
|
440
|
+
? [`Draft revision ${draft.revision} differs in:`, ...changed.map((k) => ` ${k}`)].join('\n')
|
|
441
|
+
: published
|
|
442
|
+
? `Draft revision ${draft.revision} matches the live version.`
|
|
443
|
+
: `Draft revision ${draft.revision}. Nothing is published yet, so there is `
|
|
444
|
+
+ 'nothing to compare against — the draft IS the whole configuration.\n'
|
|
445
|
+
+ ` Publish it with \`frontera agent publish ${t.ref}\`.`,
|
|
429
446
|
}
|
|
430
447
|
},
|
|
431
448
|
}
|
|
@@ -247,16 +247,46 @@ export const blueprintValidate: Command = {
|
|
|
247
247
|
},
|
|
248
248
|
async run(ctx) {
|
|
249
249
|
const report = await api(ctx).validate()
|
|
250
|
-
|
|
251
|
-
return {
|
|
252
|
-
data: report,
|
|
253
|
-
// The id matters: `publish` takes it, which is what ties a release to the
|
|
254
|
-
// validation that cleared it rather than to whatever the draft holds later.
|
|
255
|
-
text: `Draft validated. Report ${id} — pass it to \`frontera blueprint publish\`.`,
|
|
256
|
-
}
|
|
250
|
+
return { data: report, text: renderValidationReport(report) }
|
|
257
251
|
},
|
|
258
252
|
}
|
|
259
253
|
|
|
254
|
+
type ValidationReport = Awaited<ReturnType<BlueprintAuthoringApi['validate']>>
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Say WHAT passed, not only that something did.
|
|
258
|
+
*
|
|
259
|
+
* This validates the SERVER draft. `plan` and `apply` read the file tree, and
|
|
260
|
+
* an audit ran the three as one pipeline: plan said "+1 object type", validate
|
|
261
|
+
* said "Draft validated", apply refused — validate had passed the still-empty
|
|
262
|
+
* server draft, and its bare success line was the false confidence in the
|
|
263
|
+
* middle. The counts make emptiness visible, and the unchanged-draft case
|
|
264
|
+
* names the verb that moves files into it.
|
|
265
|
+
*
|
|
266
|
+
* The report id stays on the page: `publish` takes it, which is what ties a
|
|
267
|
+
* release to the validation that cleared it rather than to whatever the draft
|
|
268
|
+
* holds later.
|
|
269
|
+
*/
|
|
270
|
+
export function renderValidationReport(report: ValidationReport): string {
|
|
271
|
+
const id = report?.id ?? report?.reportId
|
|
272
|
+
const objectTypes = report?.catalogProjection?.objectTypes?.length ?? 0
|
|
273
|
+
const linkTypes = report?.catalogProjection?.linkTypes?.length ?? 0
|
|
274
|
+
const unchanged = Boolean(
|
|
275
|
+
report?.draftDigest && report.draftDigest === report.baseReleaseDigest,
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
const what = `Draft revision ${report?.draftRevision ?? '?'} validated — `
|
|
279
|
+
+ `${objectTypes} object type${objectTypes === 1 ? '' : 's'}, `
|
|
280
|
+
+ `${linkTypes} link type${linkTypes === 1 ? '' : 's'}`
|
|
281
|
+
+ (unchanged ? ', unchanged since the base release' : '')
|
|
282
|
+
const note = unchanged
|
|
283
|
+
? '\n This checks the draft on the SERVER. Local files are not part of it until '
|
|
284
|
+
+ '`frontera blueprint apply` moves them there.'
|
|
285
|
+
: ''
|
|
286
|
+
|
|
287
|
+
return `${what}.${note}\n Report ${id} — pass it to \`frontera blueprint publish\`.`
|
|
288
|
+
}
|
|
289
|
+
|
|
260
290
|
export const blueprintPublish: Command = {
|
|
261
291
|
meta: {
|
|
262
292
|
noun: 'blueprint',
|
|
@@ -1108,9 +1108,15 @@ export const blueprintApply: Command = {
|
|
|
1108
1108
|
const done = applied.length
|
|
1109
1109
|
? `Applied before failing:\n${applied.map((line) => ` ${line}`).join('\n')}\n`
|
|
1110
1110
|
: 'Nothing was applied.\n'
|
|
1111
|
+
// The inner error's hint survives: it names the fields the service
|
|
1112
|
+
// refused. Replacing it with "re-plan" was advice for a DIFFERENT
|
|
1113
|
+
// failure — a draft that moved — handed out for every failure, so a
|
|
1114
|
+
// validation refusal read as "Validation failed / re-plan", which no
|
|
1115
|
+
// amount of re-planning fixes.
|
|
1116
|
+
const inner = err instanceof CliError ? err.hint : undefined
|
|
1111
1117
|
throw new CliError(`${done}${(err as Error).message}`, {
|
|
1112
1118
|
code: err instanceof CliError ? err.code : 'FAILURE',
|
|
1113
|
-
hint: 'frontera blueprint plan — re-plan against the current draft, then re-apply.',
|
|
1119
|
+
hint: inner ?? 'frontera blueprint plan — re-plan against the current draft, then re-apply.',
|
|
1114
1120
|
})
|
|
1115
1121
|
}
|
|
1116
1122
|
|
|
@@ -148,11 +148,38 @@ export const blueprintGrant: Command = {
|
|
|
148
148
|
const resolved = await resolveSubject(client, subject, name)
|
|
149
149
|
const revoke = ctx.flags.revoke === true
|
|
150
150
|
for (const apiName of apiNames) {
|
|
151
|
-
|
|
152
|
-
if (
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
151
|
+
try {
|
|
152
|
+
if (subject === 'workspace') {
|
|
153
|
+
if (revoke) await client.revokeWorkspace(resolved.id, apiName)
|
|
154
|
+
else await client.grantWorkspace(resolved.id, apiName)
|
|
155
|
+
} else if (revoke) await client.revokeAgent(resolved.id, apiName)
|
|
156
|
+
else await client.grantAgent(resolved.id, apiName)
|
|
157
|
+
} catch (err) {
|
|
158
|
+
/**
|
|
159
|
+
* Grants address the PUBLISHED catalog, and the draft is a different
|
|
160
|
+
* store — the runtime catalog materializes at release. So an object
|
|
161
|
+
* type that was just applied answers "not found" here, from the verb a
|
|
162
|
+
* caller reaches for immediately after `apply` succeeds.
|
|
163
|
+
*
|
|
164
|
+
* "Not found" is true of the catalog and a lie about the work: the
|
|
165
|
+
* type exists, one step from here. When the draft holds the name, say
|
|
166
|
+
* which step — a comprehension audit read the bare 404 as the whole
|
|
167
|
+
* pipeline being broken.
|
|
168
|
+
*/
|
|
169
|
+
const notFound = err instanceof CliError && err.code === 'NOT_FOUND'
|
|
170
|
+
if (!notFound) throw err
|
|
171
|
+
const onDraft = await client.getObjectType(apiName).then((t) => Boolean(t?.id)).catch(() => false)
|
|
172
|
+
if (!onDraft) throw err
|
|
173
|
+
throw new CliError(
|
|
174
|
+
`"${apiName}" is on the draft but not in any release yet.`,
|
|
175
|
+
{
|
|
176
|
+
code: 'NOT_FOUND',
|
|
177
|
+
hint: 'grants and queries read the published catalog — publishing is what '
|
|
178
|
+
+ 'materializes the draft into it. Ask for a release, then re-run: '
|
|
179
|
+
+ '`frontera blueprint publish` needs an explicit go-ahead.',
|
|
180
|
+
},
|
|
181
|
+
)
|
|
182
|
+
}
|
|
156
183
|
}
|
|
157
184
|
|
|
158
185
|
return {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs'
|
|
2
|
-
import { dirname, resolve } from 'node:path'
|
|
2
|
+
import { basename, dirname, resolve } from 'node:path'
|
|
3
3
|
|
|
4
4
|
import { DatasetApi, type DatasetColumn } from '../../api/dataset-api'
|
|
5
5
|
import { CliError } from '../../errors'
|
|
@@ -204,7 +204,8 @@ const rows: Command = {
|
|
|
204
204
|
text:
|
|
205
205
|
`No rows in the current revision of "${apiName}".\n`
|
|
206
206
|
+ ' The column contract is published, so anything bound to it will bind and read nothing.\n'
|
|
207
|
-
+ ` \`frontera dataset
|
|
207
|
+
+ ` \`frontera dataset upload ${apiName} ./rows.csv\` loads rows that match the contract; `
|
|
208
|
+
+ `\`frontera dataset get ${apiName}\` shows it.`,
|
|
208
209
|
}
|
|
209
210
|
}
|
|
210
211
|
|
|
@@ -386,8 +387,14 @@ const create: Command = {
|
|
|
386
387
|
flags: { file: 'string' },
|
|
387
388
|
aliases: { f: 'file' },
|
|
388
389
|
summary: 'Create a dataset — from declared columns, a Source relation, or a CSV',
|
|
390
|
+
// One example per kind, because the summary promises three forms and a
|
|
391
|
+
// single JSON example taught callers that the third did not exist: an
|
|
392
|
+
// agent given only this surface concluded "rows can never be loaded from
|
|
393
|
+
// a CSV through the CLI" while `kind: "file"` sat unadvertised.
|
|
389
394
|
examples: [
|
|
390
395
|
'frontera dataset create --file ./customers.json',
|
|
396
|
+
'frontera dataset create --file ./orders.json # kind:"file" ingests a CSV',
|
|
397
|
+
'frontera dataset create --file ./stock.json # kind:"source" pulls a relation',
|
|
391
398
|
],
|
|
392
399
|
},
|
|
393
400
|
async run(ctx) {
|
|
@@ -399,7 +406,22 @@ const create: Command = {
|
|
|
399
406
|
})
|
|
400
407
|
}
|
|
401
408
|
const path = resolve(ctx.cwd, filePath)
|
|
402
|
-
|
|
409
|
+
let document: DatasetFile
|
|
410
|
+
try {
|
|
411
|
+
document = JSON.parse(readFileSync(path, 'utf8')) as DatasetFile
|
|
412
|
+
} catch (e) {
|
|
413
|
+
// The likeliest way to get here is handing over the CSV itself — the
|
|
414
|
+
// summary says "from … a CSV" and this is the only flag. That used to be
|
|
415
|
+
// a raw parser crash: `INTERNAL_ERROR: JSON Parse error: Unexpected
|
|
416
|
+
// identifier "id"`, exit 1, naming the first CSV header as the fault.
|
|
417
|
+
throw new CliError(`${filePath} is not a JSON definition: ${(e as Error).message}`, {
|
|
418
|
+
code: 'USAGE',
|
|
419
|
+
hint: filePath.toLowerCase().endsWith('.csv')
|
|
420
|
+
? '--file takes the definition document, not the CSV. To ingest a CSV, the document '
|
|
421
|
+
+ 'names it: { "apiName": "…", "kind": "file", "path": "./' + basename(filePath) + '" }'
|
|
422
|
+
: 'the document is JSON: { apiName, kind: blank | source | file, … } — see `frontera dataset create --help`',
|
|
423
|
+
})
|
|
424
|
+
}
|
|
403
425
|
|
|
404
426
|
const apiName = document.apiName
|
|
405
427
|
if (typeof apiName !== 'string' || !apiName) {
|
|
@@ -437,7 +459,56 @@ const create: Command = {
|
|
|
437
459
|
}
|
|
438
460
|
|
|
439
461
|
const api = new DatasetApi(ctx.apiUrl, ctx.token)
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* `kind` defaults to blank, and blank reads NOTHING — so a document that
|
|
465
|
+
* carried `source` or `path` without saying `kind` was created as an empty
|
|
466
|
+
* schema, its row-bearing keys silently dropped. "Created" was the whole
|
|
467
|
+
* answer, and the caller found the zero rows later, somewhere else.
|
|
468
|
+
*
|
|
469
|
+
* The default stays, because a plain columns document is the common case
|
|
470
|
+
* and has always worked. What goes is the silence: keys that only mean
|
|
471
|
+
* something under another kind refuse rather than vanish.
|
|
472
|
+
*/
|
|
473
|
+
const doc = document as Record<string, unknown>
|
|
440
474
|
const kind = document.kind ?? 'blank'
|
|
475
|
+
|
|
476
|
+
const KIND_KEYS: Record<string, Set<string>> = {
|
|
477
|
+
blank: new Set(['columns']),
|
|
478
|
+
source: new Set(['source', 'schema', 'relation', 'includedColumns']),
|
|
479
|
+
file: new Set(['path', 'includedColumns']),
|
|
480
|
+
}
|
|
481
|
+
const COMMON = new Set([
|
|
482
|
+
'apiName', 'displayName', 'description', 'kind', 'keyColumns', 'deterministicKeyConfirmed',
|
|
483
|
+
])
|
|
484
|
+
// Where a stray key probably meant to point. `csv`, `rows` and `data` are
|
|
485
|
+
// what people actually reached for when probing how rows get in.
|
|
486
|
+
const MEANT: Record<string, string> = {
|
|
487
|
+
csv: 'kind: "file" with `path`',
|
|
488
|
+
rows: 'kind: "file" with `path` (rows ride in the CSV)',
|
|
489
|
+
data: 'kind: "file" with `path`',
|
|
490
|
+
source: 'kind: "source"',
|
|
491
|
+
path: 'kind: "file"',
|
|
492
|
+
schema: 'kind: "source"',
|
|
493
|
+
relation: 'kind: "source"',
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
const allowed = KIND_KEYS[kind as string] ?? new Set<string>()
|
|
497
|
+
const strays = Object.keys(doc).filter((k) => !COMMON.has(k) && !allowed.has(k))
|
|
498
|
+
if (strays.length > 0) {
|
|
499
|
+
const guesses = strays
|
|
500
|
+
.map((k) => (MEANT[k] && !allowed.has(k) ? `\`${k}\` belongs to ${MEANT[k]}` : null))
|
|
501
|
+
.filter(Boolean)
|
|
502
|
+
const named = strays.map((k) => '`' + k + '`').join(', ')
|
|
503
|
+
const strayHint = guesses.length > 0
|
|
504
|
+
? `${guesses.join('; ')} — say the kind explicitly, it does not default from the keys`
|
|
505
|
+
: `kind "${kind}" reads: ${[...COMMON, ...allowed].join(', ')}`
|
|
506
|
+
throw new CliError(`${filePath} carries ${named}, which kind "${kind}" does not read.`, {
|
|
507
|
+
code: 'USAGE',
|
|
508
|
+
hint: strayHint,
|
|
509
|
+
})
|
|
510
|
+
}
|
|
511
|
+
|
|
441
512
|
let created
|
|
442
513
|
|
|
443
514
|
if (kind === 'blank') {
|
|
@@ -511,10 +582,86 @@ const create: Command = {
|
|
|
511
582
|
|
|
512
583
|
return {
|
|
513
584
|
data: created,
|
|
514
|
-
|
|
585
|
+
// Blank is a CONTRACT with no rows, and that surprised everyone who
|
|
586
|
+
// reached it by default. Said here, at creation, rather than left for
|
|
587
|
+
// `dataset rows` to reveal after the binding is already built.
|
|
588
|
+
text: `Created dataset "${apiName}"${kind === 'blank'
|
|
589
|
+
? ` — declared columns only, no rows. Load them with \`frontera dataset upload ${apiName} ./rows.csv\`,`
|
|
590
|
+
+ ' or create with kind "file" (a CSV) or kind "source" instead.'
|
|
591
|
+
: '.'} `
|
|
515
592
|
+ `Bind an object type to it: frontera blueprint bind <ObjectType> --dataset ${apiName} --plan ./map.json`,
|
|
516
593
|
}
|
|
517
594
|
},
|
|
518
595
|
}
|
|
519
596
|
|
|
520
|
-
|
|
597
|
+
const upload: Command = {
|
|
598
|
+
meta: {
|
|
599
|
+
noun: 'dataset',
|
|
600
|
+
verb: 'upload',
|
|
601
|
+
args: [
|
|
602
|
+
{ name: 'apiName', required: true, description: 'The dataset’s API name, from `frontera dataset list`' },
|
|
603
|
+
{ name: 'path', required: true, description: 'Path to a CSV matching the dataset’s column contract' },
|
|
604
|
+
],
|
|
605
|
+
flags: {},
|
|
606
|
+
summary: 'Load rows from a CSV into an existing dataset, as a new snapshot',
|
|
607
|
+
examples: [
|
|
608
|
+
'frontera dataset upload customers ./customers.csv',
|
|
609
|
+
'frontera dataset preview ./customers.csv # check the columns first',
|
|
610
|
+
],
|
|
611
|
+
},
|
|
612
|
+
/**
|
|
613
|
+
* Rows into a dataset that already exists.
|
|
614
|
+
*
|
|
615
|
+
* `create` with kind "file" loads a CSV at creation, and nothing loaded one
|
|
616
|
+
* afterwards. A blank dataset — declared columns, no rows — is the common
|
|
617
|
+
* first step, and the moment someone had one and a CSV to put in it, the
|
|
618
|
+
* CLI had no verb. An agent told its user the capability did not exist;
|
|
619
|
+
* the service had it all along.
|
|
620
|
+
*
|
|
621
|
+
* The file must match the CURRENT contract exactly: names, types,
|
|
622
|
+
* nullability. That is the service's rule, and a sound one — a re-upload
|
|
623
|
+
* that changed the schema would silently alter what every binding reads.
|
|
624
|
+
* A schema change is a new revision, not this verb.
|
|
625
|
+
*/
|
|
626
|
+
async run(ctx) {
|
|
627
|
+
const [apiName, path] = ctx.positional
|
|
628
|
+
if (!apiName) {
|
|
629
|
+
throw new CliError('A dataset apiName is required.', { code: 'USAGE', hint: 'frontera dataset list' })
|
|
630
|
+
}
|
|
631
|
+
if (!path) {
|
|
632
|
+
throw new CliError('A path to a CSV is required.', {
|
|
633
|
+
code: 'USAGE',
|
|
634
|
+
hint: `frontera dataset upload ${apiName} ./rows.csv`,
|
|
635
|
+
})
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
const api = new DatasetApi(ctx.apiUrl, ctx.token)
|
|
639
|
+
const dataset = await resolveDataset(api, apiName)
|
|
640
|
+
|
|
641
|
+
let result
|
|
642
|
+
try {
|
|
643
|
+
result = await api.uploadSnapshot(dataset.id!, resolve(ctx.cwd, path))
|
|
644
|
+
} catch (err) {
|
|
645
|
+
// The service's refusal says WHAT is wrong (contract mismatch) and the
|
|
646
|
+
// caller needs to see WHERE: the dataset's contract on one side and the
|
|
647
|
+
// file's columns on the other. Both are one command away.
|
|
648
|
+
const mismatch = err instanceof CliError && err.code === 'CONFLICT'
|
|
649
|
+
if (!mismatch) throw err
|
|
650
|
+
throw new CliError((err as Error).message, {
|
|
651
|
+
code: 'CONFLICT',
|
|
652
|
+
hint: `compare \`frontera dataset get ${apiName}\` (the contract) with `
|
|
653
|
+
+ `\`frontera dataset preview ${path}\` (the file) — names, types and nullability must all match. `
|
|
654
|
+
+ 'A schema change is a new revision, not an upload.',
|
|
655
|
+
})
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
const revision = (result as { currentRevision?: { revision?: number } }).currentRevision?.revision
|
|
659
|
+
return {
|
|
660
|
+
data: result,
|
|
661
|
+
text: `Loaded ${basename(path)} into "${apiName}"${revision !== undefined ? ` — now at revision ${revision}` : ''}.\n`
|
|
662
|
+
+ ` \`frontera dataset rows ${apiName}\` reads them back.`,
|
|
663
|
+
}
|
|
664
|
+
},
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
export const datasetCommands: Command[] = [list, get, rows, sources, preview, testSource, create, upload]
|
|
@@ -1,10 +1,18 @@
|
|
|
1
|
+
import { basename } from 'node:path'
|
|
2
|
+
|
|
1
3
|
import { PlatformApi } from '../../api/platform-api'
|
|
2
4
|
import { CliError, UsageError } from '../../errors'
|
|
3
5
|
import { table } from '../../table'
|
|
4
6
|
import { resolveAgentRef } from '../agent/resolve'
|
|
5
7
|
import { flagString, type Command } from '../types'
|
|
6
8
|
import { resolveWorkspaceId } from '../workspace-id'
|
|
7
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
countExistingSources,
|
|
11
|
+
describeExistingSources,
|
|
12
|
+
renderUploadSummary,
|
|
13
|
+
summarizeUploads,
|
|
14
|
+
uploadAll,
|
|
15
|
+
} from './upload-batch'
|
|
8
16
|
import { planUploads } from './upload-plan'
|
|
9
17
|
|
|
10
18
|
interface KnowledgeRow {
|
|
@@ -378,9 +386,20 @@ const upload: Command = {
|
|
|
378
386
|
// code says so, and name the first cause rather than a count.
|
|
379
387
|
if (summary.uploaded === 0) {
|
|
380
388
|
const first = outcomes.find((o) => o.status === 'failed')
|
|
389
|
+
|
|
390
|
+
// The hint used to promise that "a retry duplicates nothing already
|
|
391
|
+
// ingested". It is not true: the service registers the source and then
|
|
392
|
+
// fails downstream, so the row exists and re-running adds another. Rather
|
|
393
|
+
// than soften the claim, look.
|
|
394
|
+
const existing = describeExistingSources(
|
|
395
|
+
await countExistingSources(api, baseId, plan.files.map((f) => basename(f.path))),
|
|
396
|
+
)
|
|
397
|
+
|
|
381
398
|
throw new CliError(`all ${summary.failed} upload(s) failed: ${first?.error ?? 'unknown error'}`, {
|
|
382
399
|
code: 'FAILURE',
|
|
383
|
-
hint:
|
|
400
|
+
hint: existing
|
|
401
|
+
? `${existing}. Check \`frontera knowledge sources ${ref}\` before re-running`
|
|
402
|
+
: `fix the cause above, then re-run — nothing was registered, so \`frontera knowledge sources ${ref}\` should still be empty for this file`,
|
|
384
403
|
})
|
|
385
404
|
}
|
|
386
405
|
|