@alvera-ai/platform-sdk 0.12.1 → 0.13.0-next.gf3d2a4a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agent/AGENTS.md +121 -19
- package/.agent/account_management.md +79 -38
- package/.agent/action_logs.md +144 -0
- package/.agent/action_status_updaters.md +237 -53
- package/.agent/ai_agents.md +149 -53
- package/.agent/ai_sandbox.md +11 -4
- package/.agent/async.md +16 -6
- package/.agent/connected_apps.md +116 -4
- package/.agent/cookbook/_fixtures/README.md +9 -9
- package/.agent/cookbook/_fixtures/{payment_risk/_compliance_screenings_payment_risk_compliance_screening.liquid → payments/_compliance_screenings_payments_compliance_screening.liquid} +1 -1
- package/.agent/cookbook/_fixtures/{payment_risk/_compliance_screenings_payment_risk_mdm.liquid → payments/_compliance_screenings_payments_mdm.liquid} +1 -1
- package/.agent/cookbook/_fixtures/{payment_risk/_payment_accounts_payment_risk_mdm.liquid → payments/_payment_accounts_payments_mdm.liquid} +1 -1
- package/.agent/cookbook/_fixtures/{payment_risk/_payment_accounts_payment_risk_payment_account.liquid → payments/_payment_accounts_payments_payment_account.liquid} +1 -1
- package/.agent/cookbook/_fixtures/{accounts_receivable/_customers_accounts_receivable_customer.liquid → subscription/_customers_subscription_customer.liquid} +1 -1
- package/.agent/cookbook/_fixtures/{accounts_receivable/_customers_accounts_receivable_mdm.liquid → subscription/_customers_subscription_mdm.liquid} +1 -1
- package/.agent/cookbook/_setup/foundation.md +24 -4
- package/.agent/cookbook/_setup/healthcare.md +24 -4
- package/.agent/cookbook/_setup/{payment_risk.md → payments.md} +51 -31
- package/.agent/cookbook/_setup/{accounts_receivable.md → subscription.md} +51 -31
- package/.agent/cookbook/action-status-updaters.md +25 -18
- package/.agent/cookbook/bulk-ingest.md +15 -15
- package/.agent/cookbook/dunning-sms-for-delinquent.md +21 -21
- package/.agent/cookbook/{custom-tables.md → generic-tables.md} +44 -41
- package/.agent/cookbook/invite-team.md +19 -13
- package/.agent/cookbook/kyc-notification-on-account-activation.md +26 -26
- package/.agent/cookbook/marketing-campaign-send.md +1003 -0
- package/.agent/cookbook/rest-fetch.md +9 -9
- package/.agent/cookbook/sanctions-screening-with-agent-review.md +22 -22
- package/.agent/cookbook/system-templates.md +15 -15
- package/.agent/cookbook/triage-prospects-by-priority.md +17 -17
- package/.agent/cookbook/welcome-sms-for-customers.md +21 -21
- package/.agent/data_activation_clients.md +177 -42
- package/.agent/data_sources.md +55 -37
- package/.agent/datalakes.md +30 -17
- package/.agent/debugging.md +48 -14
- package/.agent/errors.md +39 -17
- package/.agent/generic_tables.md +150 -51
- package/.agent/interoperability_contracts.md +114 -15
- package/.agent/mdm.md +171 -93
- package/.agent/messages.md +252 -0
- package/.agent/templates.md +21 -4
- package/.agent/tool-call-configs.md +1 -1
- package/.agent/tools.md +150 -60
- package/.agent/type_naming.md +79 -18
- package/.agent/workflows.md +408 -62
- package/LICENSE +21 -93
- package/README.md +2 -2
- package/dist/bin/platform-sdk.mjs +53 -1
- package/dist/bin/platform-sdk.mjs.map +1 -1
- package/dist/index.d.mts +582 -66
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +654 -105
- package/dist/index.mjs.map +1 -1
- package/package.json +10 -6
- /package/.agent/cookbook/_fixtures/{accounts_receivable → subscription}/stripe_customers_batch1.csv +0 -0
|
@@ -51,6 +51,43 @@ const { data: created } = await api.dataActivationClients.create(
|
|
|
51
51
|
|
|
52
52
|
## 2. Rules the type cannot encode
|
|
53
53
|
|
|
54
|
+
### Every dataset already has a default manual-upload DAC — reuse it, never create one
|
|
55
|
+
|
|
56
|
+
Datalake provisioning auto-creates, for **every dataset** — the built-in resource
|
|
57
|
+
types (`customer`, `patient`, `legal_entity`, …) **and every generic table you add** —
|
|
58
|
+
a default manual-upload Data Activation Client, alongside the default data source +
|
|
59
|
+
manual-upload tool and a default identity interoperability template. The plumbing to
|
|
60
|
+
push rows into any dataset **already exists**; you do not create a DAC to seed test
|
|
61
|
+
data or ingest a handful of rows.
|
|
62
|
+
|
|
63
|
+
Reinventing it — e.g. calling `dataActivationClients.create(...)` inside a contract
|
|
64
|
+
hook — is an anti-pattern. That DAC then lives **outside** `infra.toml`: untracked,
|
|
65
|
+
never shown in `plan`, and not idempotent. Because DAC names are unique per datalake,
|
|
66
|
+
it **collides on the second run** — any drift or update verdict re-fires the hook,
|
|
67
|
+
which re-creates the same-named DAC and 409s (`name must be unique` — uniqueness
|
|
68
|
+
collisions are 409 Conflict, see `errors.md`), halting the apply.
|
|
69
|
+
|
|
70
|
+
The default DAC name is deterministic — `"<Datalake Name> <resource> DataActivationClient"`
|
|
71
|
+
with `is_default: true` (e.g. `"Subscription Lake customer DataActivationClient"`).
|
|
72
|
+
Discover it and ingest through it:
|
|
73
|
+
|
|
74
|
+
```typescript
|
|
75
|
+
const { data: clients } = await api.dataActivationClients.list(tenantSlug, datalakeSlug)
|
|
76
|
+
// `list` returns the paged envelope { data, meta } — the DAC array is clients.data
|
|
77
|
+
const seedDac = (clients.data ?? []).find(
|
|
78
|
+
(c) => c.is_default && c.name.endsWith('customer DataActivationClient'),
|
|
79
|
+
)
|
|
80
|
+
// ingest is SLUG-keyed (see §6) — pass seedDac.slug, not seedDac.id
|
|
81
|
+
await api.dataActivationClients.ingest(tenantSlug, datalakeSlug, seedDac.slug, { data: row })
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`ingest` is safe to re-run — MDM dedupe collapses repeat rows on the identifier
|
|
85
|
+
triple — so a hook that *ingests through the default DAC* stays idempotent across
|
|
86
|
+
re-applies. **Declare your own DAC in `infra.toml` only for a real pipeline** (an
|
|
87
|
+
SFTP/REST fetch, a cron schedule, a custom-contract binding) — never merely to push
|
|
88
|
+
a few rows in. See `interoperability_contracts.md` (born-red cures: prove the mapping
|
|
89
|
+
with `.run(...)`, seed through the default DAC).
|
|
90
|
+
|
|
54
91
|
### `tool_call.tool_call_type` must align with the bound tool's `tool_body_type`
|
|
55
92
|
|
|
56
93
|
The activation client's `tool_call` discriminator selects which
|
|
@@ -60,15 +97,22 @@ call shape will be sent to the tool at run time. The valid
|
|
|
60
97
|
|
|
61
98
|
```
|
|
62
99
|
tool_body_type: 'manual_upload' → tool_call_type: 'manual_upload'
|
|
63
|
-
tool_body_type: '
|
|
64
|
-
tool_body_type: '
|
|
65
|
-
tool_body_type: '
|
|
100
|
+
tool_body_type: 'rest_api' → tool_call_type: 'restapi_request'
|
|
101
|
+
tool_body_type: 'sftp' → tool_call_type: 'sftp_request'
|
|
102
|
+
tool_body_type: 's3' → tool_call_type: 's3_request'
|
|
103
|
+
tool_body_type: 'sql_database' → tool_call_type: 'sql_query'
|
|
104
|
+
tool_body_type: 'sharepoint' → tool_call_type: 'microsoft_share_point_excel_request'
|
|
105
|
+
tool_body_type: 'aws_lambda' → tool_call_type: 'aws_lambda_request'
|
|
66
106
|
```
|
|
67
107
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
`
|
|
71
|
-
|
|
108
|
+
The complete `tool_call_type` discriminator set is exactly those
|
|
109
|
+
seven values (`data_activation_client.ex` polymorphic embed) — there
|
|
110
|
+
is no `sms`/`sns` call shape here, because a DAC only ever binds a
|
|
111
|
+
tool whose `intent` is `data_exchange` (enforced in the changeset's
|
|
112
|
+
`validate_tool_intent`). (See `tools.md` §5 for the polymorphic call
|
|
113
|
+
shape per tool body type.) A mismatch — e.g. a `manual_upload` tool
|
|
114
|
+
bound with an `sftp_request` call — is rejected at create time as a
|
|
115
|
+
422 on `/tool_call/tool_call_type`.
|
|
72
116
|
|
|
73
117
|
### `interoperability_contract_ids` is a virtual many-to-many array
|
|
74
118
|
|
|
@@ -152,11 +196,22 @@ row_filter optional string — Liquid pre-filter; see §2
|
|
|
152
196
|
for inverted semantics
|
|
153
197
|
downstream_connection_ids optional UUID[] — DACs triggered after this
|
|
154
198
|
one completes; cycles rejected at create
|
|
199
|
+
filter_config optional embed — TemplateConfig; server-pinned
|
|
200
|
+
output_schema (enum ["true","false"]). A row-
|
|
201
|
+
level filter evaluated per fetched row
|
|
202
|
+
response_extractor optional embed — TemplateConfig ({ type, body });
|
|
203
|
+
a Liquid template that unwraps a nested API
|
|
204
|
+
envelope into the row array (e.g.
|
|
205
|
+
`{{ msg.data | to_json }}`). Load-bearing for
|
|
206
|
+
`rest_api` fetch DACs; see `rest-fetch.md` §004
|
|
155
207
|
```
|
|
156
208
|
|
|
157
209
|
**Write-only (Request-only).** None — every caller-supplied
|
|
158
210
|
field round-trips on the response (the virtual array
|
|
159
211
|
round-trips under `interoperability_contracts` on response).
|
|
212
|
+
`filter_config` / `response_extractor` come back under the read-only
|
|
213
|
+
`SimpleTemplateConfigResponse` shape (their `output_schema` is
|
|
214
|
+
server-pinned and not echoed).
|
|
160
215
|
|
|
161
216
|
## 4. Error envelopes
|
|
162
217
|
|
|
@@ -165,6 +220,7 @@ Standard JSON:API envelopes per `errors.md`. Common rejections:
|
|
|
165
220
|
| `source.pointer` | Cause |
|
|
166
221
|
|-------------------------------------------|----------------------------------------------------|
|
|
167
222
|
| `/tool_id` / `/data_source_id` | resource doesn't exist (or wrong datalake) |
|
|
223
|
+
| `/data_source_id` | does not match the bound tool's `data_source_id` — a DAC's source must equal its tool's source (source-match) |
|
|
168
224
|
| `/tool_call/tool_call_type` | mismatch with the tool's `tool_body_type` |
|
|
169
225
|
| `/interoperability_contract_ids/0` | contract id doesn't exist or wrong datalake |
|
|
170
226
|
| `/name` | uniqueness within datalake (POST and PUT — rename can collide) |
|
|
@@ -182,11 +238,24 @@ of the runtime verbs in §6.
|
|
|
182
238
|
### Read shapes
|
|
183
239
|
|
|
184
240
|
```
|
|
185
|
-
.list(tenantSlug, datalakeSlug)
|
|
186
|
-
|
|
187
|
-
.
|
|
241
|
+
.list(tenantSlug, datalakeSlug, query?) Paged: { data, meta }
|
|
242
|
+
(full Flop query — type_naming.md)
|
|
243
|
+
.get(tenantSlug, datalakeSlug, id) One row, by the UUID id (NOT slug)
|
|
244
|
+
.metadata(tenantSlug, datalakeSlug, query?) Markdown catalog page of the
|
|
245
|
+
datalake's DACs (Flop-paged)
|
|
246
|
+
.metadataDetails(tenantSlug, datalakeSlug, id)
|
|
247
|
+
Markdown of one DAC's connected-
|
|
248
|
+
dataset field docs, by UUID id
|
|
188
249
|
```
|
|
189
250
|
|
|
251
|
+
**`.get` takes the UUID `id`, not the slug.** The CRUD reads/writes
|
|
252
|
+
(`.get` / `.update` / `.delete` / `.metadataDetails`) are id-keyed —
|
|
253
|
+
the controller resolves them through the `fetch_client_by_id` plug
|
|
254
|
+
(`get_client_by_id_for_datalake!`). Only the *runtime* verbs (§6:
|
|
255
|
+
`.ingest` / `.ingestFile` / `.runManually` / `.logs.*`) are slug-keyed
|
|
256
|
+
(`fetch_client_by_slug`). To go from a name/slug to an id, `list()`
|
|
257
|
+
and match on `.slug` (or `.name`), then use the row's `id`.
|
|
258
|
+
|
|
190
259
|
### Update
|
|
191
260
|
|
|
192
261
|
`update()` replaces the whole resource — resupply the full
|
|
@@ -194,25 +263,27 @@ body; there's no partial update. Pointer fields
|
|
|
194
263
|
(`tool_id`, `data_source_id`, `interoperability_contract_ids`)
|
|
195
264
|
can be re-pointed.
|
|
196
265
|
|
|
197
|
-
**The `
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
266
|
+
**The `update()` lookup key is the UUID `id`, which never changes.**
|
|
267
|
+
But **the `slug` regenerates from `name` whenever `name` changes** —
|
|
268
|
+
the changeset re-derives it (`SlugGenerator.name_to_slug`), so the
|
|
269
|
+
response body carries a NEW slug after a rename even though you
|
|
270
|
+
addressed the row by its stable `id`. Examples:
|
|
201
271
|
|
|
202
272
|
```
|
|
203
|
-
pre-update: name: 'Original', slug: 'original'
|
|
273
|
+
pre-update: name: 'Original', slug: 'original', id: <uuid>
|
|
204
274
|
PUT body: name: 'Renamed', ...
|
|
205
|
-
PUT URL: .../data-activation-clients
|
|
206
|
-
PUT response: name: 'Renamed', slug: 'renamed'
|
|
275
|
+
PUT URL: .../data-activation-clients/<uuid> ← id is stable, addressable throughout
|
|
276
|
+
PUT response: name: 'Renamed', slug: 'renamed' ← new slug derived from the new name
|
|
207
277
|
```
|
|
208
278
|
|
|
209
|
-
**Consequence**:
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
279
|
+
**Consequence**: because the runtime verbs (§6) are slug-keyed,
|
|
280
|
+
a rename moves the ingest / run / logs endpoints to the new slug —
|
|
281
|
+
and any downstream resource that references the DAC by slug
|
|
282
|
+
(workflow templates, action bodies referencing `{{ ... }}` paths
|
|
283
|
+
derived from the DAC slug, monitoring dashboards, etc.) breaks the
|
|
284
|
+
moment the DAC is renamed. Capture the post-update slug from the
|
|
285
|
+
response body. If you rename a DAC mid-deployment, update every
|
|
286
|
+
binding that references it.
|
|
216
287
|
|
|
217
288
|
An `update()` that keeps `name` unchanged regenerates to the
|
|
218
289
|
same slug (slugification is deterministic), so cosmetic-only
|
|
@@ -220,9 +291,13 @@ updates don't break bindings.
|
|
|
220
291
|
|
|
221
292
|
### Delete
|
|
222
293
|
|
|
223
|
-
`delete()` removes the activation client.
|
|
224
|
-
|
|
225
|
-
|
|
294
|
+
`delete()` removes the activation client (by UUID `id`). Two
|
|
295
|
+
rejections: a **default DAC** (`is_default: true`) is refused with a
|
|
296
|
+
**422** on `/is_default` ("Default data activation clients cannot be
|
|
297
|
+
deleted"), and a DAC still **referenced by another resource** is
|
|
298
|
+
refused with a **409**. Otherwise it deletes; a run already in flight
|
|
299
|
+
keeps its own lifecycle (its `batch_id` is already stamped on the
|
|
300
|
+
enqueued jobs) and completes independently of the deleted row.
|
|
226
301
|
|
|
227
302
|
## 6. Runtime
|
|
228
303
|
|
|
@@ -292,6 +367,7 @@ const { data: job } = await api.dataActivationClients.ingestFile(
|
|
|
292
367
|
)
|
|
293
368
|
// job.key — echoes link.key
|
|
294
369
|
// job.job_id — numeric id of the enqueued background job
|
|
370
|
+
// job.status — initial Oban job state (e.g. 'scheduled' / 'available')
|
|
295
371
|
```
|
|
296
372
|
|
|
297
373
|
This path is **fully async** — the response acknowledges that
|
|
@@ -312,11 +388,14 @@ verb:
|
|
|
312
388
|
const { data } = await api.dataActivationClients.runManually(
|
|
313
389
|
tenantSlug, datalakeSlug, activationClientSlug,
|
|
314
390
|
)
|
|
391
|
+
// data.batch_id — UUID; returned immediately (202), the batch stamped on this run's jobs
|
|
315
392
|
```
|
|
316
393
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
394
|
+
Unlike `.ingestFile`, `.runManually` **returns the `batch_id`
|
|
395
|
+
directly** (`202` + `{ batch_id }` — `RunManuallyResponse`); you do
|
|
396
|
+
NOT have to diff the logs to find it. The actual fetch still runs
|
|
397
|
+
asynchronously in the background, so poll the logs (§6.4) with that
|
|
398
|
+
`batch_id` to know when rows have landed.
|
|
320
399
|
|
|
321
400
|
#### One-shot `tool_call` override
|
|
322
401
|
|
|
@@ -349,14 +428,21 @@ rows_ingested number — count of top-level rows the contract emitted
|
|
|
349
428
|
(cascade rows carry 0)
|
|
350
429
|
dataset_updated number — count of dataset rows whose batch_id this run wrote
|
|
351
430
|
(load-bearing: see Gotcha 7)
|
|
352
|
-
output_files
|
|
353
|
-
|
|
431
|
+
output_files { object_key, mode }[] — merged ndjson archive refs, one per
|
|
432
|
+
bucket; mode is 'regulated' | 'unregulated'
|
|
433
|
+
(empty until the post-batch merge worker commits).
|
|
434
|
+
NOT a string[] — each entry is an embed object;
|
|
435
|
+
use /logs/:id/download to fetch a presigned URL
|
|
354
436
|
```
|
|
355
437
|
|
|
356
438
|
```typescript
|
|
357
|
-
// Paginated list of log rows for an activation client.
|
|
439
|
+
// Paginated list of log rows for an activation client. Takes the full
|
|
440
|
+
// Flop query (page / page_size / order_by / order_directions / filters
|
|
441
|
+
// — type_naming.md); without it, page 1 only, and a busy client's older
|
|
442
|
+
// batches drop out of view.
|
|
358
443
|
const { data } = await api.dataActivationClients.logs.list(
|
|
359
444
|
tenantSlug, datalakeSlug, activationClientSlug,
|
|
445
|
+
{ page: 1, page_size: 50, order_by: ['inserted_at'], order_directions: ['desc'] },
|
|
360
446
|
)
|
|
361
447
|
// data.data — array of log rows; filter by batch_id to find this run
|
|
362
448
|
|
|
@@ -386,6 +472,30 @@ durable" signal — not just the presence of the log row.
|
|
|
386
472
|
|
|
387
473
|
### 6.5 Verifying ingestion: the two-step dataset search
|
|
388
474
|
|
|
475
|
+
> **Simplest landing check — `executeSql`.** Ingestion is async (the row
|
|
476
|
+
> commits a moment after the `202`). When you just need to know a row
|
|
477
|
+
> *landed*, poll `datalakes.executeSql` with a read-only `SELECT … WHERE
|
|
478
|
+
> batch_id = '…'`: **each call is a fresh read, so there is no stale
|
|
479
|
+
> snapshot to recover from** — unlike a `createUserSearch` created before
|
|
480
|
+
> the row commits, which captures nothing and never updates. Gate on it,
|
|
481
|
+
> then run the two-step `createUserSearch` below **once** (race-free) only
|
|
482
|
+
> if you need the dataset's alias-join + tokenization semantics.
|
|
483
|
+
>
|
|
484
|
+
> ```typescript
|
|
485
|
+
> const { data: probe } = await api.datalakes.executeSql(tenantSlug, datalakeSlug, {
|
|
486
|
+
> sql: `SELECT 1 FROM patients WHERE batch_id = '${batchId}' LIMIT 1`,
|
|
487
|
+
> mode: 'unregulated',
|
|
488
|
+
> })
|
|
489
|
+
> // `data` is an array-of-arrays (a `string` only for { format: 'csv' });
|
|
490
|
+
> // a non-empty page proves the batch landed. Reference the unregulated
|
|
491
|
+
> // table by its base name — `mode: 'regulated'` uses the `regulated_` prefix.
|
|
492
|
+
> const landed = typeof probe !== 'string' && probe.data.length > 0
|
|
493
|
+
> ```
|
|
494
|
+
>
|
|
495
|
+
> `createUserSearch` is the heavier, workflow-oriented surface — reach for
|
|
496
|
+
> it when you need its alias joins (`ri.value`, tokenized read-back), not
|
|
497
|
+
> for a plain "did it land" check.
|
|
498
|
+
|
|
389
499
|
The canonical way to **prove a row landed in the datalake** is
|
|
390
500
|
to query the dataset directly. This is a two-step pattern:
|
|
391
501
|
|
|
@@ -432,20 +542,25 @@ const { data: page } = await api.datasets.search(
|
|
|
432
542
|
#### SQL aliases
|
|
433
543
|
|
|
434
544
|
`createUserSearch` accepts a WHERE-clause fragment against a
|
|
435
|
-
fixed set of table aliases. Each dataset has a
|
|
436
|
-
|
|
437
|
-
|
|
545
|
+
fixed set of table aliases. Each dataset has a base alias for its
|
|
546
|
+
primary regulated table, plus `ri` for the regulated_identifiers
|
|
547
|
+
join. **The base alias is assigned by that dataset's server-side
|
|
548
|
+
decomposed query — it is NOT derivable from the dataset name, and
|
|
549
|
+
different datasets reuse the same letters.** Verified examples:
|
|
438
550
|
|
|
439
551
|
```
|
|
440
|
-
patient → rp (regulated_patients)
|
|
552
|
+
patient → rp (regulated_patients) + ri
|
|
441
553
|
appointment → ra (regulated_appointments) + ri
|
|
442
|
-
customer →
|
|
443
|
-
... (one alias per regulated dataset)
|
|
554
|
+
customer → ra (regulated_customers) + ri ← 'ra', NOT 'rc'
|
|
444
555
|
```
|
|
445
556
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
557
|
+
`customer` is a live trap: its primary table is aliased `ra`, not
|
|
558
|
+
`rc` (both `rest-fetch.md` §006 and `bulk-ingest.md` §008 query it as
|
|
559
|
+
`ra.batch_id`). Don't guess the alias from the dataset name — confirm
|
|
560
|
+
it from the dataset's green cookbook/test before writing the fragment.
|
|
561
|
+
The `ri.value` predicate filters on external identifiers (the values
|
|
562
|
+
your inbound rows carried); the `<base_alias>.batch_id` predicate
|
|
563
|
+
scopes the search to one run.
|
|
449
564
|
|
|
450
565
|
#### Tokenized vs plain column shape
|
|
451
566
|
|
|
@@ -557,3 +672,23 @@ first.
|
|
|
557
672
|
migration time) are protected. Filter them out
|
|
558
673
|
(`!d.is_default`) when listing for management purposes; you
|
|
559
674
|
can still PUT-update them (e.g. to bind additional contracts).
|
|
675
|
+
|
|
676
|
+
14. **A never-landing ingest (`dataset_updated` never flips)
|
|
677
|
+
fails silently — diagnose UPSTREAM, at the extractor, not the
|
|
678
|
+
merge.** Distinct from the trigger *lag* in §7.7: here the row
|
|
679
|
+
is *received* but never *persists*. The per-row `DataExtractor`
|
|
680
|
+
is where it dies; the later `BatchMergeWorker` "No files found
|
|
681
|
+
… *.ndjson" is a DOWNSTREAM SYMPTOM (the extractor never wrote
|
|
682
|
+
the ndjson) — don't stop there. Usual extractor causes: (a)
|
|
683
|
+
**MDM subject not resolved** — `:mdm_output_required` /
|
|
684
|
+
"upsert_dataset called without %MDMOutput{}": the contract's
|
|
685
|
+
`mdm_input_config` rendered nil (a `null`/misprovisioned
|
|
686
|
+
identity contract, or missing MDM identity fields), so a
|
|
687
|
+
domain that requires entity resolution rejects the row; (b) a
|
|
688
|
+
**per-row changeset rejection** — most often an out-of-set
|
|
689
|
+
enum (see `interoperability_contracts.md` §7). The caller sees
|
|
690
|
+
only `rows_ingested`. Diagnose with the platform's Oban error
|
|
691
|
+
logs (isolate by `batch_id`/`datalake_slug` metadata) + an
|
|
692
|
+
`execute-sql` probe of the DAC log's `dataset_updated`; the
|
|
693
|
+
CLI's `run-dac --json` + `execute-sql` verbs reproduce and
|
|
694
|
+
probe it without building an app.
|
package/.agent/data_sources.md
CHANGED
|
@@ -42,8 +42,11 @@ const { data: created } = await api.dataSources.create(
|
|
|
42
42
|
```
|
|
43
43
|
|
|
44
44
|
The body is flat — no discriminator. Every data source carries
|
|
45
|
-
the same
|
|
46
|
-
|
|
45
|
+
the same caller-supplied fields — four required (`name`, `uri`,
|
|
46
|
+
`status`, `is_default`) plus two optional (`description`,
|
|
47
|
+
`image_url`) — and the universal server-derived set (`id`,
|
|
48
|
+
`slug`, `created_at`, `updated_at`). The cookbooks omit
|
|
49
|
+
`image_url` (a five-field body is the common shape).
|
|
47
50
|
|
|
48
51
|
## 2. Rules the type cannot encode
|
|
49
52
|
|
|
@@ -98,7 +101,8 @@ multi-step UI flows.
|
|
|
98
101
|
```
|
|
99
102
|
name required string — agent-facing label
|
|
100
103
|
uri required string — endpoint identifier (free-form)
|
|
101
|
-
description optional string — narrative
|
|
104
|
+
description optional string — narrative (nullable)
|
|
105
|
+
image_url optional string — icon/image URL for the source (nullable)
|
|
102
106
|
status required enum — lifecycle state
|
|
103
107
|
is_default required bool — at most one true per datalake
|
|
104
108
|
```
|
|
@@ -123,12 +127,22 @@ Standard JSON:API envelopes per `errors.md`. Common rejections:
|
|
|
123
127
|
| `/uri` | empty/structurally invalid OR uniqueness within |
|
|
124
128
|
| | datalake (uri collision) |
|
|
125
129
|
| `/status` | value not in the typed enum |
|
|
126
|
-
| `/is_default` | another source already marked default |
|
|
127
130
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
131
|
+
A structurally invalid `/name` or `/uri` (Layer 1) is a 422; a
|
|
132
|
+
`/name` or `/uri` **uniqueness collision** (Layer 2, mapped via
|
|
133
|
+
`unique_constraint`) comes back as a **409 Conflict** — same
|
|
134
|
+
`AlveraApiError` envelope, different status (see `errors.md` for the
|
|
135
|
+
422-vs-409 split). The SDK does not validate client-side.
|
|
136
|
+
|
|
137
|
+
**The `is_default` "one default per datalake" rule is the
|
|
138
|
+
exception — it is NOT a clean 422.** The rule is enforced only by a
|
|
139
|
+
Postgres partial unique index; the changeset has no matching
|
|
140
|
+
`unique_constraint` and the controller has no `Ecto.ConstraintError`
|
|
141
|
+
rescue, so creating a second `is_default: true` source raises an
|
|
142
|
+
**unhandled 500**, not a `/is_default` 422. Don't rely on a tidy
|
|
143
|
+
envelope here — read back the datalake's existing default first (§2)
|
|
144
|
+
and pass `is_default: false` unless you intend to be the one default.
|
|
145
|
+
(The distinct *delete*-the-default guard IS a real 422 — see §5.)
|
|
132
146
|
|
|
133
147
|
## 5. Lifecycle
|
|
134
148
|
|
|
@@ -150,12 +164,13 @@ const { data: source } = await api.dataSources.create(
|
|
|
150
164
|
### Read shapes
|
|
151
165
|
|
|
152
166
|
```
|
|
153
|
-
.list(tenantSlug, datalakeSlug)
|
|
167
|
+
.list(tenantSlug, datalakeSlug, query?) Paged list:
|
|
154
168
|
{ data: DataSourceResponse[], meta }
|
|
155
|
-
|
|
169
|
+
(full Flop query — type_naming.md)
|
|
170
|
+
.get(tenantSlug, datalakeSlug, id) One row — UUID only
|
|
156
171
|
.metadata(tenantSlug, datalakeSlug) Markdown string — catalog
|
|
157
172
|
.metadataDetails(tenantSlug, datalakeSlug, Markdown string — one source
|
|
158
|
-
|
|
173
|
+
id) (UUID only)
|
|
159
174
|
```
|
|
160
175
|
|
|
161
176
|
The two `metadata` methods return `string` — agent-facing
|
|
@@ -179,34 +194,34 @@ const { data: updated } = await api.dataSources.update(
|
|
|
179
194
|
)
|
|
180
195
|
```
|
|
181
196
|
|
|
182
|
-
### Delete
|
|
197
|
+
### Delete
|
|
183
198
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
admin console.
|
|
199
|
+
```typescript
|
|
200
|
+
await api.dataSources.delete(tenantSlug, datalakeSlug, sourceId)
|
|
201
|
+
```
|
|
188
202
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
resource:
|
|
203
|
+
`delete()` removes the row and returns `204 No Content` (UUID
|
|
204
|
+
`id` only, like `.get` / `.update`). The platform applies
|
|
205
|
+
asymmetric cascade behavior by dependent resource:
|
|
192
206
|
|
|
193
207
|
- All data activation clients bound via `data_source_id`
|
|
194
|
-
cascade-delete with the source.
|
|
195
|
-
- Workflow actions referencing the data source
|
|
196
|
-
|
|
197
|
-
|
|
208
|
+
cascade-delete with the source (FK `on_delete: :delete_all`).
|
|
209
|
+
- Workflow actions referencing the data source have a
|
|
210
|
+
restricting FK (`on_delete: :restrict`), so the delete is
|
|
211
|
+
**rejected** while any action still points at the source —
|
|
212
|
+
detach those actions first.
|
|
198
213
|
|
|
199
214
|
The auto-created default Manual Upload source (see §2 "A
|
|
200
215
|
default Manual Upload data source is auto-created per
|
|
201
|
-
datalake") is **not removable
|
|
202
|
-
|
|
203
|
-
|
|
216
|
+
datalake") is **not removable**: `delete_data_source` guards
|
|
217
|
+
`is_default` and returns a 422 with `is_default` → "Default data
|
|
218
|
+
sources cannot be deleted". The default exists for the lifetime
|
|
219
|
+
of its parent datalake.
|
|
204
220
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
the source row stays in the datalake.
|
|
221
|
+
If you want to retire a source without deleting it (e.g. the
|
|
222
|
+
default, or one a workflow action still references), mark it
|
|
223
|
+
`status: 'inactive'` via PUT — the row stays in the datalake but
|
|
224
|
+
is short-circuited from new ingestion runs.
|
|
210
225
|
|
|
211
226
|
## 6. Gotchas
|
|
212
227
|
|
|
@@ -219,12 +234,15 @@ the source row stays in the datalake.
|
|
|
219
234
|
multiple datalakes, iterate by datalake or use cross-datalake
|
|
220
235
|
discovery via a higher-level surface.
|
|
221
236
|
|
|
222
|
-
2. **`is_default` collisions
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
237
|
+
2. **`is_default` collisions raise a 500, not a clean 422.** The
|
|
238
|
+
"one default per datalake" rule lives only in a Postgres partial
|
|
239
|
+
unique index — unmapped by the changeset and unrescued by the
|
|
240
|
+
controller (see §4) — so creating a second `is_default: true`
|
|
241
|
+
source currently surfaces as an unhandled 500. The SDK does NOT
|
|
242
|
+
silently demote the existing default either. Check the datalake's
|
|
243
|
+
current default and pass `is_default: false` unless you mean to
|
|
244
|
+
own it; don't branch on a `/is_default` error envelope that
|
|
245
|
+
doesn't arrive cleanly.
|
|
228
246
|
|
|
229
247
|
3. **Data sources don't carry credentials.** A source is a
|
|
230
248
|
*named endpoint*, not an authenticated connection.
|
package/.agent/datalakes.md
CHANGED
|
@@ -11,7 +11,7 @@ two databases under one roof:
|
|
|
11
11
|
Every other resource in the platform (data sources, tools, AI
|
|
12
12
|
agents, workflows, etc.) scopes under a datalake. A tenant can own
|
|
13
13
|
many datalakes; the canonical pattern is one per data domain (e.g.
|
|
14
|
-
`healthcare`, `foundation`, `
|
|
14
|
+
`healthcare`, `foundation`, `payments`).
|
|
15
15
|
|
|
16
16
|
```typescript
|
|
17
17
|
import type { PlatformApi } from '@alvera-ai/platform-sdk'
|
|
@@ -123,7 +123,7 @@ import type { DatalakeRequestWritable } from '@alvera-ai/platform-sdk'
|
|
|
123
123
|
// identical across them; only `data_domain` and the names it
|
|
124
124
|
// interpolates differ.
|
|
125
125
|
const dataDomain = 'healthcare'
|
|
126
|
-
// ^ swap to 'foundation' | '
|
|
126
|
+
// ^ swap to 'foundation' | 'subscription' | 'payments'
|
|
127
127
|
|
|
128
128
|
// Secrets resolved upstream by your code (secret store, config loader, etc.)
|
|
129
129
|
const pgHost = '...'
|
|
@@ -340,7 +340,7 @@ Datalake-specific common rejections:
|
|
|
340
340
|
| `/unregulated_cloud_storage` | synchronous probe could not reach the bucket with the supplied credentials |
|
|
341
341
|
| `/unregulated_cloud_storage/access_key_id` | supplied under `auth_method: "iam_role"` (must be omitted) |
|
|
342
342
|
| `/pool_size` | out of the inclusive range 1–50 |
|
|
343
|
-
| `/timezone` | not
|
|
343
|
+
| `/timezone` | not in the closed 8-zone US whitelist (valid IANA values like `UTC` are also rejected — see gotcha 11) |
|
|
344
344
|
|
|
345
345
|
For polymorphic embeds, the `cloud_storage_type` value is the
|
|
346
346
|
first thing to inspect — it routes downstream field validation to
|
|
@@ -400,15 +400,20 @@ the datalake row — it carries `job_id` + `datalake_id` +
|
|
|
400
400
|
separately:
|
|
401
401
|
|
|
402
402
|
```typescript
|
|
403
|
-
const { data: dl } = await api.datalakes.get(tenantSlug, created.
|
|
403
|
+
const { data: dl } = await api.datalakes.get(tenantSlug, created.id)
|
|
404
404
|
// dl.status === 'new' | 'processing' | 'ready'
|
|
405
405
|
```
|
|
406
406
|
|
|
407
|
+
**`datalakes.get` keys on the UUID `id`, not the slug** (the facade
|
|
408
|
+
signature is `get(tenantSlug, id)`, and the validated setup cookbooks
|
|
409
|
+
poll with `ctx.datalakeId`). Passing the slug 422s with `Invalid
|
|
410
|
+
format. Expected :uuid`. Stash `created.id` from the create response.
|
|
411
|
+
|
|
407
412
|
Use the canonical `waitUntilReady` from `async.md`:
|
|
408
413
|
|
|
409
414
|
```typescript
|
|
410
415
|
const dl = await waitUntilReady(
|
|
411
|
-
() => api.datalakes.get(tenantSlug, created.
|
|
416
|
+
() => api.datalakes.get(tenantSlug, created.id),
|
|
412
417
|
(status) => status === 'ready' ? 'ready' : 'pending',
|
|
413
418
|
{ intervalMs: 15_000, timeoutMs: 5 * 60_000 },
|
|
414
419
|
)
|
|
@@ -539,8 +544,8 @@ returning a different shape:
|
|
|
539
544
|
|
|
540
545
|
| Method | Returns |
|
|
541
546
|
|--------------------------------------|-------------------------------------------------------------|
|
|
542
|
-
| `.list(tenantSlug)`
|
|
543
|
-
| `.get(tenantSlug,
|
|
547
|
+
| `.list(tenantSlug, query?)` | Paged list: `{ data: DatalakeResponse[], meta }` — full Flop query (type_naming.md) |
|
|
548
|
+
| `.get(tenantSlug, id)` | One row: `DatalakeResponse` — UUID only (slug 422s) |
|
|
544
549
|
| `.metadata(tenantSlug, slug)` | **Markdown string** describing the lake |
|
|
545
550
|
| `.indexMetadata(tenantSlug)` | **Markdown string** listing every lake |
|
|
546
551
|
| `.systemDatasets(tenantSlug, slug)` | `{ datasets: string[] }` — industry-built dataset names |
|
|
@@ -602,7 +607,11 @@ the SQL between the two calls.
|
|
|
602
607
|
returns the generated `sql`, the winning `model` + `provider`, and a best-effort
|
|
603
608
|
`explanation` (`null` when the explainer is unavailable). **Only the prompt + the
|
|
604
609
|
schema cross the LLM boundary — never datalake rows**, so it is safe in both modes.
|
|
605
|
-
It returns SQL for review; it does **not** execute it.
|
|
610
|
+
It returns SQL for review; it does **not** execute it. The review step is
|
|
611
|
+
load-bearing, not ceremony: the model can emit SQL that parses but fails
|
|
612
|
+
live — e.g. `WHERE some_boolean = 1` 422s on Postgres (booleans compare to
|
|
613
|
+
`true`/`false`, never `1`). Surface the SQL to a human (or a checking step)
|
|
614
|
+
and expect to edit before `executeSql`.
|
|
606
615
|
- **`executeSql`** body is `{ sql, mode, page?, page_size? }`. It runs the SQL
|
|
607
616
|
**read-only** (INSERT/UPDATE/DELETE/DDL are rejected) on the mode-appropriate
|
|
608
617
|
schema, with Flop-inspired pagination (`page_size` capped server-side). The JSON
|
|
@@ -645,10 +654,10 @@ const { data: csv } = await api.datalakes.executeSql(
|
|
|
645
654
|
const { data: created } = await api.datalakes.create(tenantSlug, body)
|
|
646
655
|
// created.status === 'new'
|
|
647
656
|
|
|
648
|
-
await api.datalakes.migrate(tenantSlug, created.slug)
|
|
657
|
+
await api.datalakes.migrate(tenantSlug, created.slug) // migrate is slug-keyed
|
|
649
658
|
|
|
650
659
|
const ready = await waitUntilReady(
|
|
651
|
-
() => api.datalakes.get(tenantSlug, created.
|
|
660
|
+
() => api.datalakes.get(tenantSlug, created.id), // get is UUID-id-keyed
|
|
652
661
|
(s) => s === 'ready' ? 'ready' : 'pending',
|
|
653
662
|
{ intervalMs: 15_000, timeoutMs: 5 * 60_000 },
|
|
654
663
|
)
|
|
@@ -745,13 +754,17 @@ const { data: csv } = await api.datalakes.executeSql(
|
|
|
745
754
|
mentions a `greater_than: 0` or `less_than_or_equal_to: 50`
|
|
746
755
|
constraint depending on which side was crossed.
|
|
747
756
|
|
|
748
|
-
11. **`timezone` is a closed
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
757
|
+
11. **`timezone` is a closed whitelist of 8 US timezones — NOT the
|
|
758
|
+
general IANA set.** Exactly: `America/New_York`, `America/Chicago`,
|
|
759
|
+
`America/Denver`, `America/Los_Angeles`, `America/Anchorage`,
|
|
760
|
+
`America/Adak`, `Pacific/Honolulu`, `America/Phoenix`. Everything
|
|
761
|
+
else is rejected on `/timezone` with `must be one of the supported
|
|
762
|
+
US timezones: …` — including values that ARE valid IANA zones
|
|
763
|
+
(`UTC`, `Etc/UTC`, `Africa/Abidjan`). Do NOT source candidates from
|
|
764
|
+
`Intl.supportedValuesOf('timeZone')`; almost all of its ~400 values
|
|
765
|
+
are rejected. The whitelist is projected as the `timezone` enum in
|
|
766
|
+
the OpenAPI schema, so the generated `DatalakeRequest` type carries
|
|
767
|
+
the legal values and TypeScript consumers get them at compile time.
|
|
755
768
|
|
|
756
769
|
12. **`auth_method` defaults to `"password"` on every DB profile.**
|
|
757
770
|
Omitting `*_auth_method` or supplying `null` is equivalent to
|