ingestron 0.17.7 → 0.17.8

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.17.8
+
+- Pin core 0.12.14. Bridge routes: a connection with `route: bridge` lands a
+  source on one platform (such as ADF) for a flow on another (such as
+  Databricks); `check` shows it as one route and suggests it when the native
+  route is billed separately. `check` now works on projects combining ADF and
+  Databricks flows.
+- `ingestron discover` drafts data contracts from what a source offers, before
+  you write them: through the portable connector (also for native and bridge
+  connections that name a package), or from a platform metadata export with
+  `--from`.
+- Official short names `stripe`, `sharepoint`, `onedrive`, `s3`, `gcs`, `sftp`,
+  `salesforce`, `hubspot` and `jira`.
+
 ## 0.17.7
 
 - Pin core 0.12.13. Connections can name a source `kind` and a `route`

package/THIRD_PARTY_NOTICES.md

@@ -75,7 +75,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ```
 
-### @ingestron/core@0.12.13
+### @ingestron/core@0.12.14
 
 Declared licence: Apache-2.0
 

package/build/cli/cli/output.js

@@ -71,7 +71,9 @@ function routeLabel(route) {
     const r = route?.reference;
     const via = route?.route === "native"
         ? `native on ${clean(route.configuration)} via ${(route.standards ?? []).map(clean).join(", ")}`
-        : `portable ${clean(route?.package ?? "connector")} on ${clean(route?.configuration)}`;
+        : route?.route === "bridge"
+            ? `bridge: lands on ${clean(route.via)}, ingests on ${clean(route.configuration)} via ${(route.standards ?? []).map(clean).join(" → ")}`
+            : `portable ${clean(route?.package ?? "connector")} on ${clean(route?.configuration)}`;
     if (!r)
         return `${via} [no reference record]`;
     const stale = Date.now() - new Date(`${r.verified}T00:00:00Z`).getTime() >
@@ -181,6 +183,24 @@ export function terminal(result, verbose = false, args = {}) {
                     .map((f) => `\n  ${clean(f.flow)}: Python ${clean(f.pythonVersion ?? "3.12")} (${f.reused ? "reused" : "prepared"})`)
                     .join("") +
                 `\nUse ingestron run status ${clean(value.id)} --verbose for environment paths and details.`);
+        case "discover":
+            return (`Discovered ${clean(value.flow)} (${clean(value.route)}):` +
+                (value.tables ?? [])
+                    .map((t) => `\n  ${clean(t.table)}: ${Number(t.fields)} fields → ${clean(t.file)}` +
+                    (t.keys?.length
+                        ? ` (source keys: ${t.keys.map(clean).join(", ")})`
+                        : "") +
+                    (t.skipped ?? [])
+                        .slice(0, 10)
+                        .map((s) => `\n    skipped ${clean(s.name)}: ${clean(s.reason)}`)
+                        .join(""))
+                    .join("") +
+                (value.files ?? []).map((f) => `\n  wrote ${clean(f)}`).join("") +
+                (value.review ?? [])
+                    .slice(0, 10)
+                    .map((r) => `\n  review: ${clean(r)}`)
+                    .join("") +
+                `\nNext: ${clean(value.next)}`);
         case "run":
         case "run_status":
             return `Run ${clean(value.id)}: ${clean(value.status)} (${clean(value.action)}).\nTarget: ${clean(value.configuration)}; flows: ${value.flows.map(clean).join(", ")}.${runQualityLine(value.result?.flows)}\nUse ingestron run status ${clean(value.id)} to inspect the receipt.`;

package/build/cli/cli/plugin-resolution.d.ts

@@ -25,12 +25,30 @@ export declare const officialPlugins: {
         readonly path: "connectors/files/connector.yaml";
         readonly tagPrefix: "files-";
     };
+    readonly gcs: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/gcs/connector.yaml";
+        readonly tagPrefix: "gcs-";
+    };
     readonly github: {
         readonly kind: "connector";
         readonly repository: "ingestron/connectors";
         readonly path: "connectors/github/connector.yaml";
         readonly tagPrefix: "github-";
     };
+    readonly hubspot: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/hubspot/connector.yaml";
+        readonly tagPrefix: "hubspot-";
+    };
+    readonly jira: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/jira/connector.yaml";
+        readonly tagPrefix: "jira-";
+    };
     readonly local: {
         readonly kind: "provider";
         readonly repository: "ingestron/provider-local";
@@ -43,6 +61,12 @@ export declare const officialPlugins: {
         readonly path: "connectors/mysql/connector.yaml";
         readonly tagPrefix: "mysql-";
     };
+    readonly onedrive: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/onedrive/connector.yaml";
+        readonly tagPrefix: "onedrive-";
+    };
     readonly oracle: {
         readonly kind: "connector";
         readonly repository: "ingestron/connectors";
@@ -55,12 +79,42 @@ export declare const officialPlugins: {
         readonly path: "connectors/postgresql/connector.yaml";
         readonly tagPrefix: "postgresql-";
     };
+    readonly s3: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/s3/connector.yaml";
+        readonly tagPrefix: "s3-";
+    };
+    readonly salesforce: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/salesforce/connector.yaml";
+        readonly tagPrefix: "salesforce-";
+    };
+    readonly sftp: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/sftp/connector.yaml";
+        readonly tagPrefix: "sftp-";
+    };
+    readonly sharepoint: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/sharepoint/connector.yaml";
+        readonly tagPrefix: "sharepoint-";
+    };
     readonly "sql-server": {
         readonly kind: "connector";
         readonly repository: "ingestron/connectors";
         readonly path: "connectors/sql-server/connector.yaml";
         readonly tagPrefix: "sql-server-";
     };
+    readonly stripe: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/stripe/connector.yaml";
+        readonly tagPrefix: "stripe-";
+    };
 };
 export declare function officialName(reference: string): {
     name: keyof typeof officialPlugins;

package/build/cli/cli/plugin-resolution.js

@@ -27,12 +27,30 @@ export const officialPlugins = {
         path: "connectors/files/connector.yaml",
         tagPrefix: "files-",
     },
+    gcs: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/gcs/connector.yaml",
+        tagPrefix: "gcs-",
+    },
     github: {
         kind: "connector",
         repository: "ingestron/connectors",
         path: "connectors/github/connector.yaml",
         tagPrefix: "github-",
     },
+    hubspot: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/hubspot/connector.yaml",
+        tagPrefix: "hubspot-",
+    },
+    jira: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/jira/connector.yaml",
+        tagPrefix: "jira-",
+    },
     local: {
         kind: "provider",
         repository: "ingestron/provider-local",
@@ -45,6 +63,12 @@ export const officialPlugins = {
         path: "connectors/mysql/connector.yaml",
         tagPrefix: "mysql-",
     },
+    onedrive: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/onedrive/connector.yaml",
+        tagPrefix: "onedrive-",
+    },
     oracle: {
         kind: "connector",
         repository: "ingestron/connectors",
@@ -57,12 +81,42 @@ export const officialPlugins = {
         path: "connectors/postgresql/connector.yaml",
         tagPrefix: "postgresql-",
     },
+    s3: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/s3/connector.yaml",
+        tagPrefix: "s3-",
+    },
+    salesforce: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/salesforce/connector.yaml",
+        tagPrefix: "salesforce-",
+    },
+    sftp: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/sftp/connector.yaml",
+        tagPrefix: "sftp-",
+    },
+    sharepoint: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/sharepoint/connector.yaml",
+        tagPrefix: "sharepoint-",
+    },
     "sql-server": {
         kind: "connector",
         repository: "ingestron/connectors",
         path: "connectors/sql-server/connector.yaml",
         tagPrefix: "sql-server-",
     },
+    stripe: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/stripe/connector.yaml",
+        tagPrefix: "stripe-",
+    },
 };
 const catalogueUrl = "https://raw.githubusercontent.com/ingestron/connectors/main/catalogue.json";
 const stable = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;

package/build/cli/cli/workflow-commands.js

@@ -185,6 +185,21 @@ export function workflowCommands(app, host) {
         ...options,
         ...(secretsFile ? { envFile: secretsFile } : {}),
     }));
+    app
+        .command("discover")
+        .description("List every field a flow's source offers and write draft contracts for review")
+        .requiredOption("--flow <id>", "Ingestion flow to discover")
+        .option("--from <file>", "Use a source catalogue or exported metadata rows instead of reading the source")
+        .option("--out <directory>", "Directory for draft contracts", "contracts")
+        .option("--local <configuration>", "Local provider configuration used for native or bridge connections")
+        .option("--accept-keys", "Mark the source's primary keys in the drafts")
+        .option("--python <path>", "Optional Python 3.12 bootstrap override")
+        .option("--secrets-file <file>", "Project-relative dotenv file; only declared secret names are passed")
+        .action(({ secretsFile, acceptKeys, ...options }) => run("discover", {
+        ...options,
+        acceptKeys: !!acceptKeys,
+        ...(secretsFile ? { envFile: secretsFile } : {}),
+    }));
     execution
         .command("status <id>")
         .description("Read the durable local run receipt")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ingestron",
-  "version": "0.17.7",
+  "version": "0.17.8",
   "type": "module",
   "description": "Build repeatable data pipelines from reviewed contracts and versioned standards.",
   "license": "Apache-2.0",
@@ -40,7 +40,7 @@
   },
   "dependencies": {
     "@clack/prompts": "1.7.0",
-    "@ingestron/core": "0.12.13",
+    "@ingestron/core": "0.12.14",
     "@modelcontextprotocol/sdk": "1.30.0",
     "commander": "15.0.0",
     "yaml": "2.9.0"