@datarails-ai/dr-cli 1.6.86 → 1.6.87

package/dist/cli.js

@@ -8,7 +8,7 @@ import { fileURLToPath as fileURLToPath3 } from "url";
 // package.json
 var package_default = {
   name: "@datarails-ai/dr-cli",
-  version: "1.6.86",
+  version: "1.6.87",
   description: "Datarails CLI",
   type: "module",
   engines: {
@@ -38,7 +38,7 @@ var package_default = {
     postinstall: "node scripts/postinstall.mjs"
   },
   dependencies: {
-    "@datarails-ai/dr-appspace-cli": "0.10.12",
+    "@datarails-ai/dr-appspace-cli": "0.10.13",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@napi-rs/keyring": "^1.3.0",
     commander: "^12.1.0",
@@ -67,7 +67,7 @@ var package_default = {
 };
 
 // src/buildInfo.ts
-var BUILD_TIME = "2026-10-01T08:01:52.638Z";
+var BUILD_TIME = "2026-10-01T08:55:18.726Z";
 function formatBuildTime(iso) {
   if (iso === "") return "";
   const d = new Date(iso);

package/dist/scopes-map.json

@@ -1,6 +1,6 @@
 {
   "version": 1,
-  "cliVersion": "1.6.86",
+  "cliVersion": "1.6.87",
   "_contract": "Resolve a command path through `aliases` (canonical prefix + alias segment -> canonical path) before any lookup. A path listed in `pendingCommands`, or whose first segment is a key of `pending`, is UNDECIDED, not unrestricted: fail closed or skip it, and never read a path's absence from `commands` as 'no requirement'. An entry carrying `denied` is UNAVAILABLE to any AGENT-SCOPED credential — one whose `scopes` claim carries at least one `resource:action` entry; a claim that is absent, empty, capability-only or wholly unparseable is UNRESTRICTED, and this mark does not apply to it — whatever else that entry says: it is not a ceiling and it is not always-available, its `denied.reason` says why and `denied.until` names the ticket that lifts it, and the absence of `requires` on it must NEVER be read as 'no requirement'. Only an entry in `commands` with an empty `requires` means always-available.",
   "commands": {
     "login": {

package/dist/skills/create-connector-lib/SKILL.md

@@ -8,9 +8,10 @@ description: Use when a vendor's connector code should be shared across Datarail
 ## Overview
 
 A lib is the vendor's code written once: `lib/<vendor>/` owns the request, auth, pagination and the
-file format, and each org's connector is two thin files that **pass their field names** plus a few
-optional parameters. The worked example is `lib/ramp/dataqueries/select_query.py`; find its thin
-connectors with `grep -rl "lib.ramp.dataqueries.select_query" --include=*.py us us2 uk ca`. Read both first.
+file format, built on the SDK kit's generic pieces, and each org's connector is two thin files that
+**pass their field names** plus a few optional parameters. The worked example is
+`lib/ramp/dataqueries/select_query.py`; find its thin connectors with
+`grep -rl "lib.ramp.dataqueries.select_query" --include=*.py us us2 uk ca`. Read both first.
 
 **REQUIRED:** `datarails-connectors-onboard-connector` for everything around the code: branch,
 publish, datasource, filebox, binding. Its memory, streaming and 4xx rules apply to lib code too.
@@ -20,7 +21,7 @@ publish, datasource, filebox, binding. Its memory, streaming and 4xx rules apply
 ```
 lib/<vendor>/
   __init__.py                    what the lib holds: each module and its output, no one who uses it
-  _core.py                       output-neutral: credentials, token, client, pagination, query vars, logger
+  _core.py                       output-neutral vendor code on kit pieces: credential keys, token mint, client setup, paging
   connections/base_connection.py the one connection: auth only, no scope or other per-customer value
   dataqueries/select_query.py    the generic engine
   tests/test_select_query.py     the engine, on synthetic data
@@ -38,7 +39,7 @@ The connector's thin dataquery subclasses `SelectQuery` and sets only data:
 | Attribute | Meaning |
 |---|---|
 | `PATH` | endpoint under the vendor base URL |
-| `WINDOW` | the endpoint's from/to parameter names, filled from the Sync-query's `startDate`/`endDate`; omit for undated datasets |
+| `WINDOW` | the endpoint's from/to parameter names, filled from the Sync-query's `startDate`/`endDate` (read them with `kit.dates.date_window`); omit for undated datasets |
 | `COLUMNS` | `(label, source)` pairs in file order |
 
 Sources are small functions in the engine module: `field("a.b.c")` (dotted path; a missing step is
@@ -58,9 +59,13 @@ parameter fails in `connect()` before the vendor is called.
   (Workato), in code, docstrings, comments, test names, fixture data or test ids. Test data is
   synthetic. Per-customer values (columns, category or custom-field names) live only in thin files.
   Before committing, grep the lib diff for the customer's name, domain and org id.
-- **Reuse before adding.** Use `_core` helpers and an existing connection that fits; never a second
-  copy. Name modules after what they do, never after who uses them. Hand the user a list of lib
-  improvements you noticed (duplicated helpers, code that belongs in `_core`), done in their own PR.
+- **Reuse before adding.** Use the SDK kit first (`from dr_integrations_sdk import kit`: logger, query
+  vars, HTTP client, CSV writer, paging, auth, dates), then `_core` helpers and an existing connection
+  that fits; never a second copy. Name modules after what they do, never after who uses them. Hand the
+  user a list of lib improvements you noticed (duplicated helpers, code that belongs in `_core`, a
+  hand-written copy of a kit helper), done in their own PR. The kit comes from the runner image at
+  runtime, not from `.sdk-ref`: publish a kit-based lib connector only once each hook's first log line
+  in that datacenter reads `INFO:dr.sdk: hook=… kit=<version>`.
 - **Never change what another org lands.** A change to a module other orgs import must be proven
   output-neutral: their tests pass, and for text-only edits, each module's AST with docstrings stripped
   equals `origin/main`'s. A lib merge cuts new versions of every connector that bundles it; none moves
@@ -68,7 +73,11 @@ parameter fails in `connect()` before the vendor is called.
 - **Take the file format from a real landed file, not from memory.** Download the file the old
   integration landed (`dr filebox download <fb> --doc-version <id>`) and profile it: BOM, line endings,
   trailing newline, how empty and null cells are written, when values are quoted, number and boolean
-  forms, timestamp formats. The engine's cell writer reproduces exactly that.
+  forms, timestamp formats. The engine's cell writer reproduces exactly that. `kit.csv.CsvWriter` gives
+  the header-once, one-block-per-page writer and takes `csv.writer` dialect kwargs
+  (`lineterminator="\n"`, `quoting=`), but writes each value as `csv.writer` does (`str()`, `None` as an
+  empty cell), so the engine still pre-formats every cell the way the landed file shows (`true`, not
+  `True`).
 - **Credentials resolve at runtime from the datasource** (`get_credentials`), never from code, tests
   or logs (`tech/security.md` §4). Check a migrated credential's shape without printing it; a stored
   header value may carry its scheme (`Basic <base64>`).
@@ -88,6 +97,6 @@ the same fetched records through the old and new code and require identical byte
 | Customer column list or category names as the lib default | Empty default in the lib; the thin file sets them |
 | "A thin connector needs no test" | A thin file that declares columns tests its header and its org-specific sources |
 | One lib connection per scope, named after an output variant (`flat`) | One parameterized connection; the thin file sets `SCOPE` |
-| `csv.writer` / `str(True)` for the file | Write cells the way the landed file shows (`""`, LF, `true`) |
+| `CsvWriter` defaults (CRLF, `str(True)`) for the file | Pass the landed file's dialect (`lineterminator="\n"`) and pre-format cells the way it shows (`""`, `true`) |
 | Artifact test that lists each client's folder | Discover thin connectors by glob on `from lib.<vendor>.` |
-| A page cap that logs and returns | The cap raises `PermanentError`; test landing exactly on it and exceeding it |
+| A page cap that logs and returns | The cap raises `PermanentError` (`kit.paging` does at `max_pages`); test landing exactly on it and exceeding it |

package/dist/skills/datarails-connectors-onboard-connector/SKILL.md

@@ -92,8 +92,11 @@ CONNECTION = "<conn>"
 - **Reuse the lib's code before adding any.** A new variant still reuses what the lib already has: the
   helpers in `_core.py` and the lib's one connection. `lib/ramp` has a single
   `connections/base_connection.py`; a thin connection subclasses it and sets `SCOPE`, never a copy. Name a new variant after its output,
-  never after the tool or customer it came from. While you're in the lib, list what could be shared or
-  cleaned up (a helper two variants duplicate, a pattern that belongs in `_core`) and give that list to
+  never after the tool or customer it came from. Generic helpers (logger, query vars, retrying client,
+  CSV writer, paging) belong in the SDK kit (*Write the connector*), not in `_core`; `_core` holds only
+  vendor-specific code. While you're in the lib, list what could be shared or
+  cleaned up (a helper two variants duplicate, a pattern that belongs in `_core`, a hand-written copy of a
+  kit helper) and give that list to
   the user as lib suggestions. Make those changes in their own PR unless the user asks otherwise.
 - **The lib is shared: it never names a client or holds a client's config or data.** No customer or org
   id in its code, docstrings or tests; no client-specific default (a customer's accounting-category
@@ -160,14 +163,15 @@ filter ignore any path whose fourth component isn't `connections`/`dataqueries`
 change neither republishes nor burns a version.
 
 Two files, both importing from `dr_integrations_sdk`. Keep imports within the flavor's `allowlist.txt`
-(stdlib always ok; base flavor = httpx + tenacity + dateutil). No manifest — name = `<vendor>__<dq>`,
+(stdlib always ok; base flavor = httpx + tenacity + dateutil; the SDK's `kit`, below, is part of
+`dr_integrations_sdk`, so every flavor allows it). No manifest — name = `<vendor>__<dq>`,
 runtime_flavor = the `RUNTIME_FLAVOR` constant, version = the git tag CI assigns on merge (a monotonic
 `vN` per connector — NOT a content hash). **No lib (Step 0)? Start from a `samples/` template — pick by
 the SOURCE's shape:**
 
 | Vendor shape | Copy | It teaches |
 |---|---|---|
-| Flat: one endpoint you page through (`/invoices?offset=`) | **`samples/sample_http_vendor/`** — the golden path | auth gate, the three config channels, `get_query_class` dispatch, the greppable logger, the non-retryable-4xx→PermanentError guard, and both pagination styles |
+| Flat: one endpoint you page through (`/invoices?offset=`) | **`samples/sample_http_vendor/`** — the golden path, built on the kit | auth gate, the three config channels, `get_query_class` dispatch, the kit logger, HTTP client (non-retryable 4xx → `PermanentError`), CSV writer and cursor paging, and both pagination styles |
 | **Fan-out / N+1**: you must drill DOWN to reach rows (`batches → headers → lines`, `invoices → lines`, `accounts → transactions`) | **`samples/sample_streaming_fanout/`** | everything above **plus** how to fetch a lot of rows concurrently while holding almost none — the bounded worker pool + bounded queue that keeps peak memory flat under the sandbox's 512 MB ceiling (see *Memory* below) |
 | **Any shape, and a second org is likely** — write it as a lib | **`lib/sample_vendor/`** + **`samples/sample_lib_connector/`** | the lib layout: output-neutral code once in `_core.py`, one-hop literals, a page cap that raises, the thin reference-only connector and a customer extension, and the lib's tests (including a runner-style artifact test) |
 
@@ -183,6 +187,29 @@ Those carry org-specific config, may lag the current SDK/runtime contract, and c
 drift — the samples are the single maintained source of truth. At most read an existing one to sanity-check
 one behavior; never use it as the starting point to copy from.
 
+**Shared helpers come from the SDK's connector kit — never copy them, from a sample or anywhere else.**
+A template gives you the structure; the logger, config, retry, CSV and paging code comes from
+`from dr_integrations_sdk import kit` (import the package, not `kit.csv`/`kit.logging` by name — they
+would shadow the stdlib). `sample_http_vendor` uses it; `sample_streaming_fanout` and `lib/sample_vendor`
+still carry hand-written `_get_logger` / `_query_vars` / retry helpers, so swap those for the kit when you
+copy them. Reference: dr-connectors-runner `docs/connector-kit.md`.
+
+| Need | Kit |
+|---|---|
+| a logger whose lines survive the sandbox | `kit.logging.get_logger("<vendor>__<dq>")` |
+| the merged config channels, required variables | `kit.config.query_vars(req)`, `kit.config.require(vars, *names, context=)` (raises `PermanentError` naming each missing one) |
+| vendor text in a log line or an error | `kit.redact.safe_snippet` / `log_safe` (redacts credentials); `one_line` for ids and cursors |
+| HTTP with retry and 4xx classification | `kit.http.client(...)`, an `httpx.AsyncClient` (every httpx kwarg passes through): retries 429 (honouring `Retry-After`), 500/502/503/504 and transport errors; 401/403 raise `AuthError`, any other 4xx or an unfollowed redirect `PermanentError`; a 429/5xx still failing after the last attempt raises `httpx.HTTPStatusError` (runtime retry). `raise_for_status=False` hands 4xx back to you |
+| the CSV header once, one block per page | `kit.csv.CsvWriter(fields).blocks(req, pages)`; `.rows` counts rows written |
+| paging that fails loudly | `kit.paging.cursor_pages` / `offset_pages`; `kit.paging.waves(items, worker, concurrency=)` |
+| credentials and tokens | `kit.auth.credential(creds, *keys, vendor=)`, `client_credentials` / `password_grant` → `Token.header()`, `basic_header`, `bearer_header` |
+| a date window | `kit.dates.date_window(vars, start_keys=, end_keys=, required=)`, `DateWindow.chunks(days)`, `parse_date` |
+
+**Publish a kit connector only where the kit is live.** At runtime the kit comes from the **runner
+image**, not from `.sdk-ref` (that pins only what CI tests against). An image with the kit prints
+`INFO:dr.sdk: hook=… correlation_id=… kit=<version>` as each hook's first log line; until that line
+shows in the target datacenter, a connector that imports `kit` fails at import there.
+
 **Batching & output — the non-negotiable pattern (get this right FIRST, before writing fetch).**
 
 **Step 0 — decide the read strategy by the source's batching support:**
@@ -206,11 +233,9 @@ Do NOT return `has_more` for an accumulated single output (e.g. a CSV). `write_o
 month landed only the final page's rows, with no header. Correct pattern — one `fetch()`, one `write_output`, stream
 the header once then one block per page (`query.py:60-67`), `has_more=False`:
 ```python
-async def blocks():
-    yield csv_header_block(FIELDS)
-    async for page in paginate(session, req):   # loop the vendor cursor INTERNALLY
-        yield csv_rows_block(FIELDS, page)       # one Azure block per page — memory stays flat
-return HookResult(output_blob_key=await self.write_output(req, blocks()), has_more=False)
+writer = kit.csv.CsvWriter(FIELDS)                 # header on batch 0 only, then one Azure block per page
+pages = kit.paging.cursor_pages(page)              # page(cursor) -> (items, next_cursor): loop the cursor INTERNALLY
+return HookResult(output_blob_key=await self.write_output(req, writer.blocks(req, pages)), has_more=False)
 ```
 
 **Memory — the 512 MB ceiling, and the one rule: never hold anything that scales with ROW COUNT.**
@@ -248,14 +273,14 @@ Four rules, each of which that bug broke:
    from, and only a generator that does the work gets it.
 2. **`asyncio.gather` over the whole work set IS the accumulation.** It returns only when every
    coroutine finishes, and returns every result at once — there is no streaming variant. Bounded
-   waves of `gather` over a handful of **pages** are fine (results are bounded); `gather` over
+   waves over a handful of **pages** are fine (`kit.paging.waves`; results are bounded); `gather` over
    every **parent** in a fan-out is a buffer wearing a concurrency costume. Use N worker tasks
    draining a work queue and pushing finished chunks into a **bounded** `asyncio.Queue` — the
    bound is what makes `await queue.put(...)` block when the uploader falls behind, carrying
    backpressure from Azure all the way to the vendor request.
 3. **Rows become bytes when their page arrives, then are forgotten.** Measured per row: dict 846 B,
-   tuple 550 B, pre-joined CSV line 289 B — but the right answer is to keep none. `csv.writer` over
-   tuples, not `DictWriter` over dicts. The only legitimate accumulation is the **work list** (which
+   tuple 550 B, pre-joined CSV line 289 B — but the right answer is to keep none. `kit.csv.CsvWriter`
+   encodes each page (dict or tuple rows) as it arrives; hold no rows past that. The only legitimate accumulation is the **work list** (which
    parents exist), bounded by parent count, not row count — and keep it as small tuples of the
    fields you need, never the vendor's response objects (journal_lines retained ~4,700 verbose
    Oracle header dicts to read 5 fields from each).
@@ -288,7 +313,8 @@ Two more sandbox ceilings worth knowing, both undocumented elsewhere:
 (`_LINE_PAGE_LIMIT` etc.) exist so a broken cursor can't loop forever — but a cap that logs a
 warning and returns what it has lands a **partial file that looks complete**. That is how
 journal_lines silently dropped 113,171 rows of a customer's month and turned into a reconciliation
-escalation. Caps must `raise PermanentError`.
+escalation. Caps must `raise PermanentError` — `kit.paging.cursor_pages` / `offset_pages` (`max_pages=`)
+do, and also fail on a cursor that does not advance and on a vendor that caps its page size below yours.
 
 Keep that single step under the runner's **2700s (45 min) hook timeout** (`HOOK_TIMEOUT_S` in
 `dr-connectors-runtime/src/runtime/activities.py`, applied **per hook**, so `fetch` gets the whole
@@ -297,15 +323,17 @@ A timeout **blocks instead of retrying** — the budget is a property of the hoo
 of the moment, so a re-run would do the same too-slow work and fail again. One kill, one reported
 failure. Levers, in order: (1) a
 **larger vendor page size** — but the vendor may cap it (**some vendors reject `pagesize` above their own limit; a legacy
-recipe used 1000**), so probe before relying on it and **log vendor 4xx** so a rejected page
-size is visible (a bare `raise_for_status` leaves no trace); (2) **bounded concurrency** inside the one
-`fetch` — for a FLAT source, waves of N≈3–5 page GETs (`asyncio.gather`), streamed in page order,
-stopping at the first short/empty page; for a **fan-out** source, a bounded worker pool feeding a
+recipe used 1000**), so probe before relying on it and make vendor 4xx visible — `kit.http.client`'s
+`PermanentError` names the status and a redacted body (a bare `raise_for_status` leaves no trace); (2) **bounded concurrency** inside the one
+`fetch` — for a FLAT source, `kit.paging.waves(offsets, worker, concurrency=N)` with N≈3–5 over offset pages,
+yielded in page order, stopping at the first short/empty page — only over a stable sort order, or pages
+shift under you (a cursor API cannot be fanned out); for a **fan-out** source, a bounded worker pool feeding a
 bounded queue instead (`samples/sample_streaming_fanout/`) — gather over every parent both buffers
 the whole result (see *Memory*) and stalls fast parents behind slow ones in the same wave, which
-measured 376s vs 192s on the same data. Either way **with per-page retry** (retry a transient
-drop/429/5xx on that one page, not the whole month) since vendors throttle under concurrency; for
-waves use `return_exceptions` so a page fetched *beyond* the last one can't fail the run. Wave
+measured 376s vs 192s on the same data. Either way **with per-page retry** since vendors throttle under
+concurrency — `kit.http.client` retries a transient drop/429/5xx on that one request, not the whole month.
+`waves` propagates the first failure, so a vendor that answers a page *beyond* the last one with an error
+rather than an empty page needs that one status handled in the worker. Wave
 concurrency is what took a 76-page GL month from ~388s sequential (timeout) to ~177s. (3) only as a
 last resort, narrow the
 sync's date window so each run is smaller. `has_more`/`next_cursor` are reserved for when Phase-2
@@ -316,12 +344,15 @@ multi-batch assembly lands — until then treat them as unusable for a single ac
   **fail-fast auth gate** — prove auth here, once. Set module-level `RUNTIME_FLAVOR = "<flavor>"` and
   `CONNECTION = <the class>`:
   ```python
-  from dr_integrations_sdk import DataIntegrationConnection, SyncRequest
+  from dr_integrations_sdk import DataIntegrationConnection, SyncRequest, kit
   RUNTIME_FLAVOR = "base"
+  log = kit.logging.get_logger("<vendor>__<dq>")
   class MyConnection(DataIntegrationConnection):
       async def connect(self, req: SyncRequest):
           creds = await self.get_credentials(req)      # {"apikey"/"clientId"...} from the datasource
-          # mint a token / open a client, then return an in-memory session object
+          api_key = kit.auth.credential(creds, "apikey", vendor="<vendor>")  # PermanentError naming the keys, never a KeyError
+          # mint a token (kit.auth.client_credentials / password_grant) if the vendor needs one
+          log.info("connect: credential resolved")     # never the secret
           return Session(...)
   CONNECTION = MyConnection                            # the runtime object (a class), NOT a string
   ```
@@ -329,16 +360,25 @@ multi-batch assembly lands — until then treat them as unusable for a single ac
   session (never re-resolving creds) and returns a `HookResult`. Set `CONNECTION = "<conn>"` (a string
   naming the sibling connection file) and `def get_query_class(name)`:
   ```python
-  from dr_integrations_sdk import DataQuery, HookResult, SyncRequest
+  from dr_integrations_sdk import DataQuery, HookResult, SyncRequest, kit
   CONNECTION = "<conn>"                                # -> connections/<conn>.py, same vendor (a STRING here)
+  log = kit.logging.get_logger("<vendor>__<dq>")
+  FIELDS = ["id", "name", "amount"]
   class MyQuery(DataQuery):
       async def fetch(self, session, req: SyncRequest) -> HookResult:
-          cfg = req.config                             # public_config ∪ query_config ∪ filebox Sync-query (cfg["query"], parsed from its JSON text) ∪ cfg["period"]
+          q = kit.config.query_vars(req)               # public_config ∪ query_config ∪ filebox Sync-query (parsed from its JSON text), platform keys dropped
           # Merge order matters: the Sync-query is applied LAST and wins any key collision. Selectors
           # (symbol/account/entity) live there so operators can see them in the UI; query_config is
           # plumbing only (base_url, caps). Never define one key in both — see step 3c.
-          # httpx GET using `session` -> rows; write via an async generator yielding bytes
-          return HookResult(output_blob_key=await self.write_output(req, gen), has_more=False)
+          (base_url,) = kit.config.require(q, "base_url", context="<vendor> <dq>")
+          writer = kit.csv.CsvWriter(FIELDS)
+          async with kit.http.client(base_url=base_url, headers=session.headers(), logger=log) as http:
+              async def page(cursor):
+                  body = (await http.get("/rows", params={"cursor": cursor} if cursor else {})).json()
+                  return body["items"], body.get("next_cursor")
+              key = await self.write_output(req, writer.blocks(req, kit.paging.cursor_pages(page)))
+          log.info("fetch done rows=%d", writer.rows)
+          return HookResult(output_blob_key=key, has_more=False)
   def get_query_class(name: str) -> type[DataQuery]:   # name = req.config["objectType"]; single-dataset -> return the sole class
       return MyQuery
   ```
@@ -396,7 +436,8 @@ The repo-root `conftest.py` carries the whole harness, so the test itself stays
    silently — and neither shows up unless you probe both.
 3. **Non-retryable vendor 4xx → `PermanentError`** (400/401/402/403/404/422), and 429/5xx *not*
    permanent. A bare exception is `verdict: retry` — five doomed attempts, ~25 min of hammering the
-   customer's vendor account, and a locked filebox.
+   customer's vendor account, and a locked filebox. `kit.http.client` provides this (401/403 as
+   `AuthError`, a `PermanentError` subclass); the test still proves the connector uses it.
 4. **Failure and cancellation hygiene.** A failing producer or consumer must surface promptly with
    no orphaned tasks and no hang; the suite runs with `filterwarnings = ["error"]` and fails any
    test that leaves a task pending.
@@ -813,8 +854,8 @@ unmerged connector has no version to ingest, bind, or run.
 | New artifact code doesn't run even after publish | The runtime resolves the **active-version pointer** (`connector`+`org`), not your latest publish. | `dr connectors publish <dq.py>` then `dr connectors promote <name> --artifact-version vN` (both need `NODE_EXTRA_CA_CERTS`). Confirm: in-cluster `GET /internal/connectors/<name>/active?org_id=<org>`. |
 | `dr filebox get` never shows the filebox `uuid` | Not exposed by that command. | Read `filebox_uuid` from the `sync.requested` event (`datarails-connect-inngest`), or capture it from `dr filebox create --json`. |
 | Creating a binding via `curl localhost:8000/connectors` (or `/connector-queries`) → `401` / SPA HTML | Those POSTs are **authenticated** now; the port-forward serves the SPA. There is no unauthenticated create path. | Use the dr-cli (carries the JWT): `dr connectors connector create --secret-ref <ds>` then `dr connectors query create --connector-id <id> --filebox-uuid <uuid> --artifact-name … --artifact-version … --query-config '{…}'`. See step 4. |
-| Connector's own `log.info(...)` lines never appear in the runner logs / Groundcover (but `dr.runner` INFO does) | The hook runs in a **sandbox whose root logger has no handler**, so bare `logging` is dropped (only WARNING+ via lastResort, and even that may not surface). | In the connector, attach an explicit `logging.StreamHandler(sys.stdout)` to your logger (idempotently), set level INFO, and format with a `[<vendor>__<dq>]` prefix so lines are greppable in the shared `connectors-runner` pod. Log auth (never the secret), the window, per-page progress, row count. |
-| A credential reaches Groundcover through a log line that logs no secret | The connector logged the raw `base_url`. It is **operator-supplied binding config**, so it can carry `user:pass@` userinfo or a credential in the query string — and the runner log is a shared multi-tenant stream. 24 connectors in dr-connectors currently do this (harmless for the keyless public sources, not for the credentialed ones). | Log only the hostname: `urlsplit(base_url).hostname` drops userinfo, port, path and query, which is all a log line needs to identify the environment. `samples/sample_streaming_fanout/` and the databricks connections model this. |
+| Connector's own `log.info(...)` lines never appear in the runner logs / Groundcover (but `dr.runner` INFO does) | The hook runs in a **sandbox whose root logger has no handler**, so bare `logging` is dropped (only WARNING+ via lastResort, and even that may not surface). | Use `kit.logging.get_logger("<vendor>__<dq>")`: a stdout handler with a `[<vendor>__<dq>]` prefix, so lines are greppable in the shared `connectors-runner` pod. Never hand-roll a `StreamHandler` helper. Log auth (never the secret), the window, per-page progress, row count. |
+| A credential reaches Groundcover through a log line that logs no secret | The connector logged the raw `base_url`. It is **operator-supplied binding config**, so it can carry `user:pass@` userinfo or a credential in the query string — and the runner log is a shared multi-tenant stream. 24 connectors in dr-connectors currently do this (harmless for the keyless public sources, not for the credentialed ones). | Log only the hostname: `urlsplit(base_url).hostname` drops userinfo, port, path and query, which is all a log line needs to identify the environment. `samples/sample_streaming_fanout/` and the databricks connections model this. `kit.http.client`'s retry and error lines already name only `METHOD scheme://host[:port]/path`, and `kit.redact.safe_snippet` redacts credential-shaped vendor text. |
 | `get_query_class` receives `None` / multi-dataset connector raises "unknown objectType" / wrong dataset runs | `objectType` was placed at the data-query **config top level** instead of **inside `config.query`**. The runtime reads `dq["query"].get("objectType")` with **no top-level fallback**, so it comes through as `None`. | Put `objectType` inside the Sync-query, which is JSON **text**: `config.query = '{"objectType":"<dataset>"}'` (drop any top-level `objectType`). `dr connectors dev-fetch` warns when it's missing. Single-dataset connectors survive via their default; multi-dataset ones fail. |
 | Sync-query written as a nested **dict** (the pre-DR-50411 shape) | `config.query` was the only vendor field held as an object. It still syncs — `_sync_query_object` accepts a dict — but the monolith's legacy filebox-settings textarea binds the value directly, so an object rendered there as the literal **`[object Object]`** and a Save wrote that string back over the query. Monolith PR https://github.com/Datarails/dr-datarails-server/pull/7390 took `DatarailsConnector` out of `list_of_integrations_with_dynamics_fields` to put it on the free-text path. | Write `config.query` as JSON text (step 3c). There is **no backfill command** — the dropped `connector_sync_query_to_text` is referenced in an older control-plane commit message but never shipped — so an existing filebox converts only when re-saved through the connectors Sync-query editor. |
 | Sync `Completed` but the landed file has only the **last page** of rows (and no CSV header) | The connector returned `has_more=True`/`next_cursor` (multi-batch pagination). `post_fetch` is fed only `batch_keys[-1]` — the last batch (Phase-2 assembly unimplemented, `activities.py:204-236`); and `write_output`'s `commit_block_list` replaces `req.output_sas` each call. So every batch overwrote the prior and only the last survived. | Paginate **inside one `fetch()`** with a single `write_output` (header once + one block per page), `has_more=False`. See §1. Verified: this is the only way to land a complete multi-page CSV on the current runtime. |
@@ -822,7 +863,7 @@ unmerged connector has no version to ingest, bind, or run.
 | Filebox versions all carry a date tag (e.g. `Month: August 2026`) but every version holds the same full snapshot | A **date tag on a filebox whose Sync-query has no date window** — a reference/dictionary dataset (payers, HCPCS, modifiers, tags, users). The tag is applied at upload from the selected period regardless of what was fetched, so it records a period the data was never filtered by. | Match the tag to the query: `--no-date` at create for an undated query; a dated tag only when the query renders a `{{ date_tag| … }}` window. On an existing filebox, drop the tag with `dr filebox set-tag-config <id> --remove <TagName>` (or recreate it with `--no-date` — note that mints a NEW `uuid`, so the `ConnectorQuery` binding must be recreated too). |
 | Confused about publish vs ingest | CI publishes to JFrog on merge (cuts the git tag `vN`); the CLI ingests a published version into a DC | `dr connectors publish <path/to/dataqueries/<dq>.py> --server DEV --account <customer-user-email> --json` reads the newest git tag (`git fetch --tags` + max `vN`, or `--tag vN`) — **no local compose** — then the control-plane **pulls that tag from JFrog** (no upload) and registers it (needs `NODE_EXTRA_CA_CERTS`); returns `{name, version: "vN", digest}`. |
 | Run `Failed`, `connect` completion `verdict: ok` but `fetch` completion `verdict: retry`, error `'async for' requires an object with __aiter__ method, got function` | Passed the async-generator **function** to `write_output` — `write_output(req, gen)` instead of **calling** it `write_output(req, gen())`. `write_output` does `async for chunk in <arg>`, which needs an async **iterator**, not a function. (The connect gate passing but fetch dying instantly is the tell.) | Call the generator: `write_output(req, gen())`. The golden template already shows `write_output(req, rows())` / `write_output(req, blocks())` — match it exactly; the missing `()` is easy to drop. |
-| Vendor **4xx** (e.g. `402 Payment Required`, `401`, `403`, bad-symbol `404`) makes the `fetch` completion `verdict: retry` and the run **retries forever**; the HTTP status shows ONLY in the completion store, never in the Inngest run `output` (which is `null`). | A bare `r.raise_for_status()` raises `httpx.HTTPStatusError` for **every** status ≥ 400, and the runtime classifies that as **transient → retry**. But a plan/auth/bad-request 4xx will never succeed on retry. | Log the status+body (skill §1), then for non-retryable statuses (`400/401/402/403/404/422`) `raise PermanentError(...)`; leave `429` + `5xx` to `raise_for_status` (genuinely transient). Concrete case: FMP's free tier answers **402** when `limit` exceeds its historical allowance (dev key caps statements at the last **5** annual records) — a small `limit` or a paid key fixes that one; the PermanentError stops the infinite loop for the general case. The golden `sample_http_vendor` template now shows this guard. |
+| Vendor **4xx** (e.g. `402 Payment Required`, `401`, `403`, bad-symbol `404`) makes the `fetch` completion `verdict: retry` and the run **retries forever**; the HTTP status shows ONLY in the completion store, never in the Inngest run `output` (which is `null`). | A bare `r.raise_for_status()` raises `httpx.HTTPStatusError` for **every** status ≥ 400, and the runtime classifies that as **transient → retry**. But a plan/auth/bad-request 4xx will never succeed on retry. | Use `kit.http.client` (skill §1): 401/403 raise `AuthError` and any other 4xx `PermanentError`, each naming the status and a redacted body; 429 and 5xx are retried, then left to the runtime's retry (genuinely transient). Concrete case: FMP's free tier answers **402** when `limit` exceeds its historical allowance (dev key caps statements at the last **5** annual records) — a small `limit` or a paid key fixes that one; the PermanentError stops the infinite loop for the general case. The golden `sample_http_vendor` template is built on this client. |
 | Connector works perfectly in local/dev testing, then in production `fetch` dies with `MemoryError`, `MemoryError: Unable to allocate output buffer`, an SSL/crypto allocation failure, or `RuntimeError: can't start new thread` — often within a second of "fetch start", and deterministically on every one of the 5 attempts | The sandbox caps **virtual address space at `RLIMIT_AS` = 512 MB** (`dr-connectors-runner/src/runner/sandbox_entry.py`), and the runner pod's limit is also 512Mi. The guard is `if sys.platform != "win32"`, so **the ceiling does not exist on a Windows dev box** — a buffering connector cannot fail locally. Two causes, usually together: (a) the fetch accumulates the result set (a `rows` list, `asyncio.gather` over every parent, a full CSV string + its `.encode()` copy) — journal_lines measured **378 MB live to emit 67 MB**, 243 MB of it row dicts at 846 B/row; (b) asyncio's **default** DNS ThreadPoolExecutor grew to `min(32, node_cores + 4)` threads × 8 MB stack ≈ **256 MB of address space that is never reclaimed**, which is why a brief concurrency spike poisons every later phase of the run. | Stream: do the fetching **inside** the generator passed to `write_output`, replace gather-over-everything with a bounded worker pool feeding a bounded `asyncio.Queue`, format rows to CSV bytes immediately (never hold dicts), and re-batch to ~4 MB blocks. Cap the executor once: `asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=4))`. Full rules in §1 *Memory*; working template `samples/sample_streaming_fanout/`. Reproduce locally in `python:3.12-slim` under the same rlimits — never trust a Windows run for this. |
 | Hook dies with no traceback and no `MemoryError`; runner reports `sandbox exited -24` | **`RLIMIT_CPU = 60s`** (`sandbox_entry.py`, right beside the memory limit) — 60 seconds of **CPU** time, then SIGXCPU/SIGKILL. Async I/O waiting is free, but TLS handshakes, gzip and JSON parsing across thousands of requests are not. | Cut CPU per row (fewer/larger vendor pages, `onlyData` where the vendor supports it, no per-row dict churn) and log `time.process_time()` next to wall clock so proximity to the cap is visible before it kills a run. |
 | A sync `Completed` and the file **looks** complete, but the customer's totals are short | A page-limit "safety cap" (`_LINE_PAGE_LIMIT`, `_HEADER_PAGE_LIMIT`, …) tripped and the code **logged a warning and returned what it had**, so a partial result landed as if whole. Same failure class as a missing offset follow-up: journal_lines silently dropped **113,171 rows** of one customer's month this way. | Caps must `raise PermanentError`, never warn-and-return — a failed sync is recoverable, a silently partial landing gets reconciled against and believed. Audit every `log.warning` that is followed by `return`. |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datarails-ai/dr-cli",
-  "version": "1.6.86",
+  "version": "1.6.87",
   "description": "Datarails CLI",
   "type": "module",
   "engines": {
@@ -17,7 +17,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@datarails-ai/dr-appspace-cli": "0.10.12",
+    "@datarails-ai/dr-appspace-cli": "0.10.13",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@napi-rs/keyring": "^1.3.0",
     "commander": "^12.1.0",