@datarails-ai/dr-cli 1.6.138 → 1.6.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +58 -7
- package/dist/scopes-map.json +1 -1
- package/dist/skills/create-connector-lib/SKILL.md +1 -1
- package/dist/skills/{datarails-connectors-onboard-connector → drip-implement-connector}/SKILL.md +14 -3
- package/dist/skills/drip-implement-connector/reference/sftp.md +246 -0
- package/dist/skills/plugins.json +1 -1
- package/dist/skills/validate-workato-migration/SKILL.md +3 -3
- package/package.json +1 -1
- package/scripts/postinstall.mjs +6 -12
package/dist/cli.js
CHANGED
|
@@ -8,7 +8,7 @@ import { fileURLToPath as fileURLToPath3 } from "url";
|
|
|
8
8
|
// package.json
|
|
9
9
|
var package_default = {
|
|
10
10
|
name: "@datarails-ai/dr-cli",
|
|
11
|
-
version: "1.6.
|
|
11
|
+
version: "1.6.140",
|
|
12
12
|
description: "Datarails CLI",
|
|
13
13
|
type: "module",
|
|
14
14
|
engines: {
|
|
@@ -69,7 +69,7 @@ var package_default = {
|
|
|
69
69
|
};
|
|
70
70
|
|
|
71
71
|
// src/buildInfo.ts
|
|
72
|
-
var BUILD_TIME = "2026-10-
|
|
72
|
+
var BUILD_TIME = "2026-10-16T10:39:49.051Z";
|
|
73
73
|
function formatBuildTime(iso) {
|
|
74
74
|
if (iso === "") return "";
|
|
75
75
|
const d = new Date(iso);
|
|
@@ -31927,7 +31927,7 @@ function registerErrorEnrichmentsCommand(root) {
|
|
|
31927
31927
|
|
|
31928
31928
|
// src/commands/skill.ts
|
|
31929
31929
|
import { createHash as createHash8 } from "crypto";
|
|
31930
|
-
import { readFile as readFile3, mkdir as mkdir2, readdir as readdir2, rm as rm2, rmdir, stat, writeFile as writeFile3 } from "fs/promises";
|
|
31930
|
+
import { lstat, readFile as readFile3, mkdir as mkdir2, readdir as readdir2, rm as rm2, rmdir, stat, writeFile as writeFile3 } from "fs/promises";
|
|
31931
31931
|
import { dirname as dirname6, join as join9, resolve as resolve2 } from "path";
|
|
31932
31932
|
|
|
31933
31933
|
// src/commands/_skillsRoot.ts
|
|
@@ -32107,6 +32107,9 @@ function skillDirAbs(targetAbs, dotDir, name) {
|
|
|
32107
32107
|
return join9(targetAbs, dotDir, "skills", name);
|
|
32108
32108
|
}
|
|
32109
32109
|
var MANIFEST_FILE = ".dr-cli-manifest.json";
|
|
32110
|
+
var RETIRED_SKILLS = {
|
|
32111
|
+
"datarails-connectors-onboard-connector": "drip-implement-connector"
|
|
32112
|
+
};
|
|
32110
32113
|
var MARKER_BEGIN = "<!-- dr-cli:begin -->";
|
|
32111
32114
|
var MARKER_END = "<!-- dr-cli:end -->";
|
|
32112
32115
|
var USERSPACE_ENV = "DR_CLI_USERSPACE_SKILLS";
|
|
@@ -32209,7 +32212,16 @@ async function runSkillInstall(cfg) {
|
|
|
32209
32212
|
const skills = await getEmbeddedSkills();
|
|
32210
32213
|
const verbose = cfg.verbose ?? false;
|
|
32211
32214
|
const probe = await pluginManagedProbe(abs, cfg.env);
|
|
32212
|
-
|
|
32215
|
+
const pluginSkip = probe.managed && !(cfg.force ?? false);
|
|
32216
|
+
await removeRetiredSkills({
|
|
32217
|
+
targetAbs: abs,
|
|
32218
|
+
skills,
|
|
32219
|
+
// On a skipped install, a replacement whose plugin is absent is not written below.
|
|
32220
|
+
servedByPlugins: pluginSkip ? probe.installedPlugins : null,
|
|
32221
|
+
out: cfg.out,
|
|
32222
|
+
verbose
|
|
32223
|
+
});
|
|
32224
|
+
if (pluginSkip) {
|
|
32213
32225
|
cfg.out.write(
|
|
32214
32226
|
`Skipped ${join9(abs, ".claude", "skills")}: this machine gets the dr-cli skills from the dr-cli plugin (${probe.reason}). Pass --force to write them anyway.
|
|
32215
32227
|
`
|
|
@@ -32350,9 +32362,34 @@ async function readManifest(skillDir) {
|
|
|
32350
32362
|
return [];
|
|
32351
32363
|
}
|
|
32352
32364
|
}
|
|
32365
|
+
async function removeRetiredSkills(cfg) {
|
|
32366
|
+
const { targetAbs, out } = cfg;
|
|
32367
|
+
for (const [old, replacement] of Object.entries(RETIRED_SKILLS)) {
|
|
32368
|
+
for (const dotDir of [".claude", ".gemini"]) {
|
|
32369
|
+
const skillDir = skillDirAbs(targetAbs, dotDir, old);
|
|
32370
|
+
const removed = await removeUserSpaceCopy(skillDir);
|
|
32371
|
+
if (removed === null) continue;
|
|
32372
|
+
const files = `${removed.files} file${removed.files === 1 ? "" : "s"}`;
|
|
32373
|
+
out.write(
|
|
32374
|
+
`Removed retired skill ${old}: ${files} at ${skillDir} (now ${replacement})` + (removed.dirRemoved ? "" : "; directory kept, it holds files the CLI did not write") + "\n"
|
|
32375
|
+
);
|
|
32376
|
+
if (dotDir !== ".claude" || cfg.servedByPlugins === null) continue;
|
|
32377
|
+
const next = cfg.skills.find((s) => s.name === replacement);
|
|
32378
|
+
if (next === void 0 || cfg.servedByPlugins.has(next.plugin)) continue;
|
|
32379
|
+
await installSkillTree({
|
|
32380
|
+
skillDir: skillDirAbs(targetAbs, dotDir, next.name),
|
|
32381
|
+
files: next.files,
|
|
32382
|
+
out,
|
|
32383
|
+
verbose: cfg.verbose,
|
|
32384
|
+
label: `skill ${next.name}`
|
|
32385
|
+
});
|
|
32386
|
+
}
|
|
32387
|
+
}
|
|
32388
|
+
}
|
|
32353
32389
|
async function removeUserSpaceCopy(skillDir) {
|
|
32354
32390
|
const manifestPath = join9(skillDir, MANIFEST_FILE);
|
|
32355
32391
|
try {
|
|
32392
|
+
if ((await lstat(skillDir)).isSymbolicLink()) return null;
|
|
32356
32393
|
await stat(manifestPath);
|
|
32357
32394
|
} catch {
|
|
32358
32395
|
return null;
|
|
@@ -32360,7 +32397,9 @@ async function removeUserSpaceCopy(skillDir) {
|
|
|
32360
32397
|
let files = 0;
|
|
32361
32398
|
for (const rel of await readManifest(skillDir)) {
|
|
32362
32399
|
try {
|
|
32363
|
-
const
|
|
32400
|
+
const segs = relSegments(rel);
|
|
32401
|
+
if (await crossesLink(skillDir, segs.slice(0, -1))) continue;
|
|
32402
|
+
const p = join9(skillDir, ...segs);
|
|
32364
32403
|
await rm2(p, { force: true });
|
|
32365
32404
|
await rmdir(dirname6(p)).catch(() => {
|
|
32366
32405
|
});
|
|
@@ -32375,6 +32414,18 @@ async function removeUserSpaceCopy(skillDir) {
|
|
|
32375
32414
|
);
|
|
32376
32415
|
return { files, dirRemoved };
|
|
32377
32416
|
}
|
|
32417
|
+
async function crossesLink(root, dirSegs) {
|
|
32418
|
+
let p = root;
|
|
32419
|
+
for (const seg of dirSegs) {
|
|
32420
|
+
p = join9(p, seg);
|
|
32421
|
+
try {
|
|
32422
|
+
if ((await lstat(p)).isSymbolicLink()) return true;
|
|
32423
|
+
} catch {
|
|
32424
|
+
return false;
|
|
32425
|
+
}
|
|
32426
|
+
}
|
|
32427
|
+
return false;
|
|
32428
|
+
}
|
|
32378
32429
|
async function installSkillTree(opts) {
|
|
32379
32430
|
const { skillDir, files, out, verbose, label } = opts;
|
|
32380
32431
|
const prev = await readManifest(skillDir);
|
|
@@ -35874,7 +35925,7 @@ function registerDocumentsCommand(root) {
|
|
|
35874
35925
|
// src/commands/integrations.ts
|
|
35875
35926
|
import { Option as Option2 } from "commander";
|
|
35876
35927
|
import { constants as fsConstants } from "fs";
|
|
35877
|
-
import { lstat, open as open2 } from "fs/promises";
|
|
35928
|
+
import { lstat as lstat2, open as open2 } from "fs/promises";
|
|
35878
35929
|
|
|
35879
35930
|
// src/api/integrations.ts
|
|
35880
35931
|
async function list23(client, opts, reqOptions) {
|
|
@@ -43860,7 +43911,7 @@ function parseSavedSearchId(raw) {
|
|
|
43860
43911
|
return id;
|
|
43861
43912
|
}
|
|
43862
43913
|
async function writeSqlFile(path13, sql) {
|
|
43863
|
-
const existing = await
|
|
43914
|
+
const existing = await lstat2(path13).catch((err) => {
|
|
43864
43915
|
if (err.code === "ENOENT") return void 0;
|
|
43865
43916
|
throw err;
|
|
43866
43917
|
});
|
package/dist/scopes-map.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 1,
|
|
3
|
-
"cliVersion": "1.6.
|
|
3
|
+
"cliVersion": "1.6.140",
|
|
4
4
|
"_contract": "Resolve a command path through `aliases` (canonical prefix + alias segment -> canonical path) before any lookup. A path listed in `pendingCommands`, or whose first segment is a key of `pending`, is UNDECIDED, not unrestricted: fail closed or skip it, and never read a path's absence from `commands` as 'no requirement'. An entry carrying `denied` is UNAVAILABLE to any AGENT-SCOPED credential — one whose `scopes` claim carries at least one `resource:action` entry; a claim that is absent, empty, capability-only or wholly unparseable is UNRESTRICTED, and this mark does not apply to it — whatever else that entry says: it is not a ceiling and it is not always-available, its `denied.reason` says why and `denied.until` names the ticket that lifts it, and the absence of `requires` on it must NEVER be read as 'no requirement'. Only an entry in `commands` with an empty `requires` means always-available.",
|
|
5
5
|
"commands": {
|
|
6
6
|
"login": {
|
|
@@ -13,7 +13,7 @@ file format, built on the SDK kit's generic pieces, and each org's connector is
|
|
|
13
13
|
`lib/ramp/dataqueries/select_query.py`; find its thin connectors with
|
|
14
14
|
`grep -rl "lib.ramp.dataqueries.select_query" --include=*.py us us2 uk ca`. Read both first.
|
|
15
15
|
|
|
16
|
-
**REQUIRED:** `
|
|
16
|
+
**REQUIRED:** `drip-implement-connector` for everything around the code: branch,
|
|
17
17
|
publish, datasource, filebox, binding. Its memory, streaming and 4xx rules apply to lib code too.
|
|
18
18
|
|
|
19
19
|
## Layout
|
package/dist/skills/{datarails-connectors-onboard-connector → drip-implement-connector}/SKILL.md
RENAMED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
---
|
|
2
|
-
name:
|
|
3
|
-
description: Use when creating a new connector for
|
|
2
|
+
name: drip-implement-connector
|
|
3
|
+
description: Use when creating a new connector for DRIP (Datarails Integration Platform, the connectors platform) and wiring it end-to-end on a dev/test env - writing the connection + dataquery files, publishing the artifact, binding it, and setting up the datasource + filebox so a sync actually runs. Triggers - "new connector", "onboard a connector", "publish a connector", "wire fmp_fx/<x> connector", "connector sync does nothing", "connector-jobs queue empty", "add <vendor> for another org", "reuse the <vendor> connector", "lib/<vendor>", "shared vendor code", "DRIP connector", "SFTP connector", "pull files from SFTP", "lib/sftp", "sftp flavor", "host_key", ConnectorQuery/binding/datasource/filebox for a DatarailsConnector.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
-
#
|
|
6
|
+
# Implement a DRIP connector (implement + wire end-to-end)
|
|
7
7
|
|
|
8
8
|
## Overview
|
|
9
9
|
|
|
10
|
+
**DRIP (Datarails Integration Platform)** is the product: the connectors platform that runs
|
|
11
|
+
customer connectors (dr-connectors code, the connectors control plane, runtime and runners) and lands
|
|
12
|
+
their output in fileboxes. This skill builds one DRIP connector and wires it until a sync lands.
|
|
13
|
+
|
|
10
14
|
A **connector** is a two-file artifact — a `connections/<conn>.py` composed with a `dataqueries/<dq>.py`
|
|
11
15
|
(no manifest; see the dr-connectors README for the authoritative file layout + programming model) —
|
|
12
16
|
published to the shared JFrog repo by CI, then pulled into a datacenter's blob on ingest. Those two
|
|
@@ -73,6 +77,10 @@ loses the audited operator→target trail (`tech/security.md` §2.2, §8).
|
|
|
73
77
|
A vendor's code may already live once in **`lib/<vendor>/`** (e.g. `lib/ramp`, `lib/sap_b1`,
|
|
74
78
|
`lib/metmuseum`). List `lib/` in `C:\codegithub\dr-connectors` before anything else.
|
|
75
79
|
|
|
80
|
+
**The source is an SFTP folder** → it is `lib/sftp` (any server: the customer's or the Datarails-hosted
|
|
81
|
+
one), on the `sftp` flavor. Follow **`reference/sftp.md`**: the thin files, a connector-specific
|
|
82
|
+
variant with its test, the datasource credentials (pinned `host_key`), the Sync-query and polling.
|
|
83
|
+
|
|
76
84
|
**The lib exists** → the connector is **two thin files**, nothing to implement:
|
|
77
85
|
|
|
78
86
|
```python
|
|
@@ -613,6 +621,8 @@ python -c "import os,json;json.dump({'name':'<name>-connector','organization':<o
|
|
|
613
621
|
python -c "import os,json;json.dump({'name':'<name>-connector','organization':<org_id>,'type':'DatarailsConnector','is_sync_service':True,'config':{'oauth':{'authorize_url':'<vendor authorize url>','token_url':'<vendor token url>','scopes':'<space-separated scopes>','client_id':os.environ['<CLIENT_ID_ENV>']},'oauth_secrets':{'client_secret':os.environ['<CLIENT_SECRET_ENV>']}}},open('ds.json','w'))"
|
|
614
622
|
# ...with nothing answered, the OAuth config is just: 'config':{'oauth':{}}
|
|
615
623
|
#
|
|
624
|
+
# SFTP — host, port, username, password and/or private_key, and the pinned host_key: reference/sftp.md §3.
|
|
625
|
+
#
|
|
616
626
|
dr integrations add --payload-file ds.json --json $ACC # -> note the datasource id
|
|
617
627
|
rm -f ds.json
|
|
618
628
|
|
|
@@ -921,3 +931,4 @@ unmerged connector has no version to ingest, bind, or run.
|
|
|
921
931
|
- `dr-cli` — datasource/filebox/integration commands.
|
|
922
932
|
- `datarails-connect-inngest` — read the sync.requested event (for `filebox_uuid`) + run status.
|
|
923
933
|
- `datarails-connectors-add-flavor` — add a flavor if the connector needs new deps.
|
|
934
|
+
- `reference/sftp.md` — SFTP connectors on `lib/sftp`: sample code, credentials, Sync-query, polling.
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
# SFTP connectors on DRIP (`lib/sftp`)
|
|
2
|
+
|
|
3
|
+
An SFTP connector lands the files a customer drops in an SFTP folder. It never needs new vendor
|
|
4
|
+
code: dr-connectors already ships **`lib/sftp`**, a provider of **`lib/filesource`**, on the `sftp`
|
|
5
|
+
runner flavor. One lib serves any server, the customer's own or the SFTP server Datarails hosts in
|
|
6
|
+
each datacenter:
|
|
7
|
+
|
|
8
|
+
| Piece | Holds |
|
|
9
|
+
|---|---|
|
|
10
|
+
| Datasource credentials | the server: `host`, `port`, `username`, `password` and/or `private_key`, the pinned `host_key` |
|
|
11
|
+
| Filebox Sync-query | which files: `folder`, `pattern`, the dating rule, the period window |
|
|
12
|
+
| Binding | `--runtime-flavor sftp`; optional `max_files` in `query_config` |
|
|
13
|
+
|
|
14
|
+
Read the lib's own docstrings before you start. They are the authoritative contract and this page
|
|
15
|
+
only points into them: `lib/sftp/__init__.py`, `lib/sftp/connections/base_connection.py`,
|
|
16
|
+
`lib/sftp/_core.py`, and `lib/filesource/dataqueries/files.py`, which has every Sync-query variable
|
|
17
|
+
and the three run modes.
|
|
18
|
+
|
|
19
|
+
**Not the same thing as `dr integrations sftp`.** That command gives a filebox a folder on the
|
|
20
|
+
Datarails-hosted SFTP server that the customer *pushes* to (the dr-cli skill's
|
|
21
|
+
`reference/integrations.md`). A DRIP SFTP connector *pulls* from a server. It can pull from the
|
|
22
|
+
hosted one too, with the org's SFTP login.
|
|
23
|
+
|
|
24
|
+
## 1. The connector: two thin files
|
|
25
|
+
|
|
26
|
+
Most SFTP connectors are just the lib (Step 0 of the skill). Layout
|
|
27
|
+
`<datacenter>/<org_id>_(<org_name>)/sftp/`:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
# <datacenter>/<org_id>_(<org_name>)/sftp/connections/<conn>.py
|
|
31
|
+
"""<org>'s SFTP login on <server>: the lib's one SFTP connection; the datasource holds the host,
|
|
32
|
+
user, password and pinned host key."""
|
|
33
|
+
|
|
34
|
+
from lib.sftp.connections.base_connection import RUNTIME_FLAVOR, SftpConnection as CONNECTION # noqa: F401
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
# <datacenter>/<org_id>_(<org_name>)/sftp/dataqueries/files.py
|
|
39
|
+
"""<org>'s SFTP files: the lib's `files` query. The folder, file rule and period window are the
|
|
40
|
+
filebox's Sync-query."""
|
|
41
|
+
|
|
42
|
+
from lib.sftp.dataqueries.files import OBJECT_TYPES, get_query_class # noqa: F401
|
|
43
|
+
|
|
44
|
+
CONNECTION = "<conn>"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Name the connection after the server or feed (`datarails_sftp`, `<bank>_sftp`). The artifact is
|
|
48
|
+
`sftp__files`. A dataquery that only re-exports the lib needs no test of its own: the lib's tests cover
|
|
49
|
+
it (`scripts/check_connector_tests.py`). `dev-1/69_(test-connectors.com)/sftp/` is the reference
|
|
50
|
+
pair, the platform's end-to-end check of the `sftp` flavor.
|
|
51
|
+
|
|
52
|
+
## 2. A connector-specific variant
|
|
53
|
+
|
|
54
|
+
When the Sync-query's regex cannot express the rule, subclass the lib's `FilesQuery` in the dataquery
|
|
55
|
+
file and override one hook. Typical cases are a date a regex cannot read, files to leave out, and
|
|
56
|
+
a file that must land in another shape. Keep `OBJECT_TYPES` one import hop away, and define your
|
|
57
|
+
own `get_query_class`:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
# <datacenter>/<org_id>_(<org_name>)/sftp/dataqueries/files.py
|
|
61
|
+
"""<org>'s <feed> on SFTP: the lib's `files` query, with the feed's own date rule and landed columns.
|
|
62
|
+
|
|
63
|
+
THE DATE. `<prefix>_<MonYYYY>.csv` (e.g. `GL_Aug2026.csv`) belongs to that month; any other name has no
|
|
64
|
+
date and is never an item.
|
|
65
|
+
|
|
66
|
+
THE FILE. Lands under the header the filebox's table expects, columns picked from the source.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
import datetime as dt
|
|
70
|
+
import re
|
|
71
|
+
|
|
72
|
+
from dr_integrations_sdk import DataQuery
|
|
73
|
+
|
|
74
|
+
from lib.filesource import _engine
|
|
75
|
+
from lib.filesource.rewrite import CsvRewrite
|
|
76
|
+
from lib.sftp.dataqueries.files import OBJECT_TYPES, FilesQuery as _FilesQuery # noqa: F401
|
|
77
|
+
|
|
78
|
+
CONNECTION = "<conn>"
|
|
79
|
+
|
|
80
|
+
_NAME_MONTH = re.compile(r"_([A-Za-z]{3})([0-9]{4})[.]csv$")
|
|
81
|
+
_LANDED = CsvRewrite(header=("Account", "Period", "Amount"), columns=(0, 2, 5), skip_records=1)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class FilesQuery(_FilesQuery):
|
|
85
|
+
def date_of(self, file: _engine.RemoteFile, selection: _engine.Selection) -> dt.date | None:
|
|
86
|
+
m = _NAME_MONTH.search(file.name)
|
|
87
|
+
if not m:
|
|
88
|
+
return None
|
|
89
|
+
try:
|
|
90
|
+
return dt.datetime.strptime(f"{m[1]}{m[2]}", "%b%Y").date()
|
|
91
|
+
except ValueError:
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
def transform(self, file, chunks, *, encoding=None):
|
|
95
|
+
# Stream it: never read a file whole. `encoding` is the Sync-query's, when set.
|
|
96
|
+
return _LANDED.stream(chunks, name=file.name, encoding=encoding)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def get_query_class(name: str) -> type[DataQuery]:
|
|
100
|
+
return FilesQuery
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The other hooks are `empty_output()`, which says what a period sync lands when no file matches
|
|
104
|
+
(the default is a PermanentError), and the `lib.filesource.rewrite` steps `decode`, `replace` and
|
|
105
|
+
`lines`. A new source format, such as a bank statement converted to CSV, is a dataquery module of
|
|
106
|
+
its own on `lib.filesource`. `lib/sftp/bai2.py` and the `cibc_bai2` connectors are the example.
|
|
107
|
+
|
|
108
|
+
A variant adds behaviour, so it needs a test next to it (`sftp/tests/test_files.py`). The `connector`
|
|
109
|
+
fixture from the repo's `conftest.py` loads the files under test:
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
# <datacenter>/<org_id>_(<org_name>)/sftp/tests/test_files.py
|
|
113
|
+
"""<org>'s <feed>: the month comes from the name, and an unreadable name has no date."""
|
|
114
|
+
|
|
115
|
+
import datetime as dt
|
|
116
|
+
|
|
117
|
+
import pytest
|
|
118
|
+
|
|
119
|
+
from lib.filesource import _engine
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def remote(name: str) -> _engine.RemoteFile:
|
|
123
|
+
at = dt.datetime(2026, 9, 3, 10, 0, tzinfo=dt.UTC)
|
|
124
|
+
return _engine.RemoteFile(path=f"/outbound/{name}", relative=name, size=10, modified=at,
|
|
125
|
+
content_key=f"10|{int(at.timestamp())}", ref={"path": f"/outbound/{name}"})
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@pytest.mark.parametrize("name, day", [
|
|
129
|
+
("GL_Aug2026.csv", dt.date(2026, 8, 1)),
|
|
130
|
+
("GL_aug2026.csv", dt.date(2026, 8, 1)),
|
|
131
|
+
("GL_2026_08.csv", None),
|
|
132
|
+
("GL_Foo2026.csv", None),
|
|
133
|
+
])
|
|
134
|
+
def test_the_month_comes_from_the_name(connector, name, day):
|
|
135
|
+
query = connector.query_class("files")()
|
|
136
|
+
assert query.date_of(remote(name), _engine.Selection(dated_by="modified")) == day
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Test a `transform` the same way: feed it chunks (an async generator of `bytes`) and compare the
|
|
140
|
+
joined output. The connectors under `ca/*/sftp/tests/` use an in-memory `Source`/`Provider` to run a
|
|
141
|
+
whole period sync or discover. Copy that pattern when the variant changes which files a sync takes.
|
|
142
|
+
Fixtures are synthetic. Never copy a customer's file into a test.
|
|
143
|
+
|
|
144
|
+
## 3. The datasource: the server login
|
|
145
|
+
|
|
146
|
+
One datasource per server login. The credentials sit in the datasource's `config`, and the lib reads
|
|
147
|
+
them by key:
|
|
148
|
+
|
|
149
|
+
| Key | |
|
|
150
|
+
|---|---|
|
|
151
|
+
| `host` | bare hostname or IP, with no scheme, user, port or path |
|
|
152
|
+
| `port` | optional, default 22 |
|
|
153
|
+
| `username` | |
|
|
154
|
+
| `password` and/or `private_key` | an OpenSSH or PEM key (Ed25519, ECDSA or RSA), plus `passphrase` when the key is encrypted |
|
|
155
|
+
| `host_key` | **required.** The server's public key: the bare `<type> <base64>`, a `known_hosts` line, or raw `ssh-keyscan` output (the first key wins). A server with an unknown or changed key is refused |
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
# Pin the server's key. Run it from a trusted network, and check the fingerprint with the server's owner.
|
|
159
|
+
ssh-keyscan -t ed25519 <host> > host_key.txt
|
|
160
|
+
|
|
161
|
+
python -c "import os,json;json.dump({'name':'<name>-sftp','organization':<org_id>,'type':'DatarailsConnector','is_sync_service':True,'config':{'host':'<host>','port':22,'username':os.environ['SFTP_USER'],'password':os.environ['SFTP_PASSWORD'],'host_key':open('host_key.txt').read()}},open('ds.json','w'))"
|
|
162
|
+
dr integrations add --payload-file ds.json --json $ACC # -> note the datasource id
|
|
163
|
+
rm -f ds.json host_key.txt
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
For key auth, swap `'password'` for `'private_key':open(os.environ['SFTP_KEY_FILE']).read()`, and add
|
|
167
|
+
`'passphrase'` when the key is encrypted. Never echo or commit any of these values.
|
|
168
|
+
|
|
169
|
+
**Only public hosts.** The lib refuses a host that resolves to a private, loopback, link-local or
|
|
170
|
+
cluster-internal address. The one exception is the Datarails-hosted SFTP server of each datacenter
|
|
171
|
+
(`HOSTED_SFTP_HOSTS` in `lib/sftp/_core.py`), which sits behind a private endpoint. Pulling from it
|
|
172
|
+
uses the customer org's SFTP login and that account's `<account>.blob.core.windows.net` host. A hosted
|
|
173
|
+
account missing from that list fails with "resolves to a private ... address". The fix is a lib PR
|
|
174
|
+
that adds it, not a different host.
|
|
175
|
+
|
|
176
|
+
## 4. The Sync-query: which files
|
|
177
|
+
|
|
178
|
+
Set it with `dr filebox data-query set` (step 3c of the skill) as JSON **text**, with `objectType`
|
|
179
|
+
inside it:
|
|
180
|
+
|
|
181
|
+
```json
|
|
182
|
+
{"objectType": "files", "folder": "/outbound", "pattern": "GL_*.csv",
|
|
183
|
+
"dated_by": "name", "date_regex": "_(?P<year>[0-9]{4})_(?P<month>[0-9]{2})[.]csv$",
|
|
184
|
+
"from": "{{ date_tag|somonth|format:'%Y-%m-%d' }}", "to": "{{ date_tag|eomonth|format:'%Y-%m-%d' }}",
|
|
185
|
+
"fileName": "GL_{{ date_tag|format:'%Y_%m' }}"}
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
`fileName` takes no extension: the runtime appends `.csv` (a `.csv` of your own would sanitize to
|
|
189
|
+
`_csv.csv`). The rules are the skill's step 3c note.
|
|
190
|
+
|
|
191
|
+
| Variable | |
|
|
192
|
+
|---|---|
|
|
193
|
+
| `folder` | required. Read only under it |
|
|
194
|
+
| `depth` | subfolder levels to list, default 0 |
|
|
195
|
+
| `pattern` | file-name glob, case-insensitive, default `*` |
|
|
196
|
+
| `dated_by` | `modified` (default), `name`, `path` (a dated folder such as `2026/08/`), or `none` |
|
|
197
|
+
| `date_regex`, `date_format` | with `name` or `path`: named groups `year`/`month`/`day`, or one group read with `date_format` |
|
|
198
|
+
| `from` / `to` | period mode window. Required unless `dated_by: none` or `latest: true` |
|
|
199
|
+
| `latest` | period mode: only the newest file of the window |
|
|
200
|
+
| `combine` | `single` (default, exactly one file, byte for byte) or `csv` (same-header CSVs as one file, with `header_rows`) |
|
|
201
|
+
| `decompress` | `auto` (default, gunzips `.gz`), `gzip` or `none` |
|
|
202
|
+
| `encoding` | passed to a variant's `transform`. Without a transform it changes nothing |
|
|
203
|
+
|
|
204
|
+
The filebox's DATE tag follows the same rule as any connector (step 3b of the skill). A dated rule
|
|
205
|
+
gets a matching tag. `dated_by: none` gets `--no-date` and no `from`/`to`. `combine: single` with no
|
|
206
|
+
file, or with more than one, is a PermanentError, as is more than `max_files` (default 500) matching
|
|
207
|
+
files.
|
|
208
|
+
|
|
209
|
+
## 5. Period sync or polled
|
|
210
|
+
|
|
211
|
+
- **Period sync**: an ordinary filebox data-query sync (step 5 of the skill). It lands the files of
|
|
212
|
+
the selected period as one version.
|
|
213
|
+
- **Polled**: the control plane lists the folder on a schedule and lands each new or changed file as
|
|
214
|
+
its own version, dated by the file's own date. This is what a feed the customer drops daily wants,
|
|
215
|
+
and what a Workato `new_file_in_dir` recipe did. Enable it after the binding exists:
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
dr connectors poll enable <filebox_id> --same-name new-version $ACC # a re-written file lands again
|
|
219
|
+
dr connectors poll get <filebox_id> $ACC
|
|
220
|
+
dr connectors poll items <filebox_id> $ACC # the item ledger
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
A polled filebox needs exactly one DATE tag. With `dated_by: name` or `path`, the window is the
|
|
224
|
+
item's own date. With `modified`, it is the file's modification time, which is the one to use when a
|
|
225
|
+
file is re-written after its period ends. `--interval`, `--lookback`, `--settle` and `--since` tune
|
|
226
|
+
the poll (`dr connectors poll enable --help`).
|
|
227
|
+
|
|
228
|
+
## 6. Bind and run
|
|
229
|
+
|
|
230
|
+
The binding is step 4 of the skill, with the `sftp` flavor:
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
dr connectors query create --connector-id <id> --filebox-uuid <uuid> --filebox-id <filebox_id> \
|
|
234
|
+
--artifact-name sftp__files --artifact-version <vN> --runtime-flavor sftp --json $ACC
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Errors you will meet in the completion store's `verdict`/`error`:
|
|
238
|
+
|
|
239
|
+
| Error | Meaning |
|
|
240
|
+
|---|---|
|
|
241
|
+
| AuthError | the login was rejected. Check `username` and `password`/`private_key` |
|
|
242
|
+
| `the datasource has no host_key` / key refused | `host_key` is missing, or the server's key changed. Re-pin it only after the owner confirms the change |
|
|
243
|
+
| `resolves to a private ... address` | the host is not public (§3) |
|
|
244
|
+
| `unknown objectType` | `objectType` is missing from the Sync-query JSON text |
|
|
245
|
+
| `exactly one file` / no file | `combine: single` matched 0 or several files. Narrow `pattern` or use `latest` |
|
|
246
|
+
| TransientError (timeout, reset) | the runtime retries it. Repeated ones mean the server is unreachable from our egress, so ask the owner to allow-list it |
|
package/dist/skills/plugins.json
CHANGED
|
@@ -3,13 +3,13 @@
|
|
|
3
3
|
"connector-sandbox",
|
|
4
4
|
"create-connector-lib",
|
|
5
5
|
"datarails-connect-workato",
|
|
6
|
-
"datarails-connectors-onboard-connector",
|
|
7
6
|
"dr-error-enrichment",
|
|
8
7
|
"dr-error-enrichment-author",
|
|
9
8
|
"dr-error-enrichment-improve",
|
|
10
9
|
"dr-error-enrichment-maintain",
|
|
11
10
|
"dr-error-enrichment-reconcile",
|
|
12
11
|
"dr-filebox-api-setup",
|
|
12
|
+
"drip-implement-connector",
|
|
13
13
|
"validate-workato-migration"
|
|
14
14
|
],
|
|
15
15
|
"dr-cli": [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: validate-workato-migration
|
|
3
|
-
description: Use when a customer's Workato feed is being migrated to a dr-connectors (DRIP) connector and the connector's output has to be proven identical to Workato's before cut-over, or when working in the Workato-migration staging org test-workato-migration.com (US 21304, US2 1691, UK 3289, CA 5106). Also for "query create" returning 422 "artifact version not registered" for a staging-org binding, and for comparing connector CSVs with Workato filebox versions. Not for building or publishing the connector itself (
|
|
3
|
+
description: Use when a customer's Workato feed is being migrated to a dr-connectors (DRIP) connector and the connector's output has to be proven identical to Workato's before cut-over, or when working in the Workato-migration staging org test-workato-migration.com (US 21304, US2 1691, UK 3289, CA 5106). Also for "query create" returning 422 "artifact version not registered" for a staging-org binding, and for comparing connector CSVs with Workato filebox versions. Not for building or publishing the connector itself (drip-implement-connector) or for reading Workato alone (datarails-connect-workato). Triggers - "validate the migration", "compare Workato vs connector", "parity check against Workato", "test the migration in the staging org", "test_workato_migration org", "are the connector files identical to Workato", "migration validation report".
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Validate a Workato → DRIP migration in the staging org
|
|
@@ -39,7 +39,7 @@ and the access rules are in [reference/staging-orgs.md](reference/staging-orgs.m
|
|
|
39
39
|
3. **Byte-identical is the only pass.** sha256 equal. Same values but different quoting,
|
|
40
40
|
`-1` vs `-1.0` or CRLF vs LF is a **FAIL**: the connector must reproduce Workato's bytes.
|
|
41
41
|
4. **Stop on mismatch.** Report and wait for the user. Don't edit, republish or re-sync on your own.
|
|
42
|
-
5. `dr connectors` calls need VPN + `NODE_EXTRA_CA_CERTS` (see `
|
|
42
|
+
5. `dr connectors` calls need VPN + `NODE_EXTRA_CA_CERTS` (see `drip-implement-connector`).
|
|
43
43
|
6. Workato is **read-only**: GET only, via `datarails-connect-workato`.
|
|
44
44
|
7. **Secrets never printed.** Credentials go from their source straight into the payload file in
|
|
45
45
|
the same shell call, and the file is deleted afterwards.
|
|
@@ -165,6 +165,6 @@ Symptoms and fixes: [reference/gotchas.md](reference/gotchas.md).
|
|
|
165
165
|
|
|
166
166
|
## Related
|
|
167
167
|
|
|
168
|
-
- `
|
|
168
|
+
- `drip-implement-connector`: build, publish and wire the connector.
|
|
169
169
|
- `datarails-connect-workato`: recipes, connections, properties (GET only).
|
|
170
170
|
- `dr-switch-user`: `dr login-as` details. `connector-sandbox`: local fix loop before a merge.
|
package/package.json
CHANGED
package/scripts/postinstall.mjs
CHANGED
|
@@ -34,18 +34,12 @@
|
|
|
34
34
|
// because npm aborts the entire install on a non-zero postinstall exit.
|
|
35
35
|
// Failing to copy a skill shouldn't break the operator's install.
|
|
36
36
|
//
|
|
37
|
-
// Cleaning up removed skills:
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
// import { join } from 'node:path';
|
|
44
|
-
// await rm(join(target, '.claude', 'skills', '<old-skill-name>'),
|
|
45
|
-
// { recursive: true, force: true });
|
|
46
|
-
//
|
|
47
|
-
// Files dropped *within* a surviving skill need no such call: each install records
|
|
48
|
-
// a `.dr-cli-manifest.json` per skill and prunes what the previous one listed.
|
|
37
|
+
// Cleaning up renamed or removed skills: add the old name to `RETIRED_SKILLS` in
|
|
38
|
+
// src/commands/skill.ts. The `dr skill install` below then removes the copy an
|
|
39
|
+
// earlier install wrote under `.claude/skills/<old>/` and `.gemini/skills/<old>/`,
|
|
40
|
+
// on plugin-managed machines too. Files dropped *within* a surviving skill need
|
|
41
|
+
// nothing: each install records a `.dr-cli-manifest.json` per skill and prunes
|
|
42
|
+
// what the previous one listed.
|
|
49
43
|
|
|
50
44
|
import { spawnSync } from 'node:child_process';
|
|
51
45
|
import { existsSync } from 'node:fs';
|