ingestron 0.17.6 → 0.17.8

package/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.17.8
+
+- Pin core 0.12.14. Bridge routes: a connection with `route: bridge` lands a
+  source on one platform (such as ADF) for a flow on another (such as
+  Databricks); `check` shows it as one route and suggests it when the native
+  route is billed separately. `check` now works on projects combining ADF and
+  Databricks flows.
+- `ingestron discover` drafts data contracts from what a source offers, before
+  you write them: through the portable connector (also for native and bridge
+  connections that name a package), or from a platform metadata export with
+  `--from`.
+- Official short names `stripe`, `sharepoint`, `onedrive`, `s3`, `gcs`, `sftp`,
+  `salesforce`, `hubspot` and `jira`.
+
+## 0.17.7
+
+- Pin core 0.12.13. Connections can name a source `kind` and a `route`
+  (`portable` or `native`); a native connection is generated by the provider's
+  own ingestion standard, with no silent fallback. `check` lists each flow's
+  source, route and reference record (licence, cost, access, maturity) and the
+  other routes available. Stale reference records are flagged.
+- Official short names `postgresql`, `mysql` and `oracle` for the new database
+  connectors.
+
 ## 0.17.6
 
 - Pin core 0.12.12. Providers can declare where each engine's rules are enforced,

package/THIRD_PARTY_NOTICES.md

@@ -75,7 +75,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ```
 
-### @ingestron/core@0.12.12
+### @ingestron/core@0.12.14
 
 Declared licence: Apache-2.0
 

package/build/cli/cli/output.js

@@ -61,6 +61,39 @@ function runQualityLine(flows) {
             .join(", ")}${warned.length > 5 ? ", …" : ""}`
         : ", all passed"}.`;
 }
+// Source routes from core (PB-064); absent with older core versions.
+const costLabel = {
+    none: "no extra cost",
+    included: "included in platform pricing",
+    "separately-billed": "billed separately",
+};
+function routeLabel(route) {
+    const r = route?.reference;
+    const via = route?.route === "native"
+        ? `native on ${clean(route.configuration)} via ${(route.standards ?? []).map(clean).join(", ")}`
+        : route?.route === "bridge"
+            ? `bridge: lands on ${clean(route.via)}, ingests on ${clean(route.configuration)} via ${(route.standards ?? []).map(clean).join(" → ")}`
+            : `portable ${clean(route?.package ?? "connector")} on ${clean(route?.configuration)}`;
+    if (!r)
+        return `${via} [no reference record]`;
+    const stale = Date.now() - new Date(`${r.verified}T00:00:00Z`).getTime() >
+        180 * 86_400_000;
+    return `${via} [${clean(r.maturity)}, ${costLabel[r.cost?.model] ?? clean(r.cost?.model)}${stale ? `, record checked ${clean(r.verified)}` : ""}]`;
+}
+function sourceLines(sources) {
+    if (!Array.isArray(sources) || !sources.length)
+        return "";
+    return ("\nSources:" +
+        sources
+            .slice(0, 20)
+            .map((s) => `\n  ${clean(s.flow)}: ${clean(s.connection)} (${clean(s.kind)}) ${routeLabel(s.selected)}` +
+            (s.alternative
+                ? `\n    Also available: ${routeLabel(s.alternative)}`
+                : ""))
+            .join("") +
+        (sources.length > 20 ? `\n  … ${sources.length - 20} more` : "") +
+        "\nUse --json for every route with documentation, licence, cost and access.");
+}
 function pluginLabel(reference) {
     for (const [name, entry] of Object.entries(officialPlugins)) {
         const prefixes = [
@@ -140,7 +173,8 @@ export function terminal(result, verbose = false, args = {}) {
             return value.mode === "draft"
                 ? `Draft configuration checked (${value.flows} flows).\n${value.pending.length} unresolved inputs. Strict build readiness has not been checked.`
                 : `Project checks passed${value.nodes === undefined ? "" : ` (${value.nodes} planned nodes)`}.\nValidation is local; no platform connection was tested.` +
-                    dataProductLines(value.dataProducts, value.quality);
+                    dataProductLines(value.dataProducts, value.quality) +
+                    sourceLines(value.sources);
         case "doctor":
             return `Project ${clean(value.project)} (${clean(value.environment)})\n${value.pending.length} unresolved inputs.\n${clean(value.platformAccess)}\nNext: ${clean(value.next)}`;
         case "runtime_prepare":
@@ -149,6 +183,24 @@ export function terminal(result, verbose = false, args = {}) {
                     .map((f) => `\n  ${clean(f.flow)}: Python ${clean(f.pythonVersion ?? "3.12")} (${f.reused ? "reused" : "prepared"})`)
                     .join("") +
                 `\nUse ingestron run status ${clean(value.id)} --verbose for environment paths and details.`);
+        case "discover":
+            return (`Discovered ${clean(value.flow)} (${clean(value.route)}):` +
+                (value.tables ?? [])
+                    .map((t) => `\n  ${clean(t.table)}: ${Number(t.fields)} fields → ${clean(t.file)}` +
+                    (t.keys?.length
+                        ? ` (source keys: ${t.keys.map(clean).join(", ")})`
+                        : "") +
+                    (t.skipped ?? [])
+                        .slice(0, 10)
+                        .map((s) => `\n    skipped ${clean(s.name)}: ${clean(s.reason)}`)
+                        .join(""))
+                    .join("") +
+                (value.files ?? []).map((f) => `\n  wrote ${clean(f)}`).join("") +
+                (value.review ?? [])
+                    .slice(0, 10)
+                    .map((r) => `\n  review: ${clean(r)}`)
+                    .join("") +
+                `\nNext: ${clean(value.next)}`);
         case "run":
         case "run_status":
             return `Run ${clean(value.id)}: ${clean(value.status)} (${clean(value.action)}).\nTarget: ${clean(value.configuration)}; flows: ${value.flows.map(clean).join(", ")}.${runQualityLine(value.result?.flows)}\nUse ingestron run status ${clean(value.id)} to inspect the receipt.`;

package/build/cli/cli/plugin-resolution.d.ts

@@ -25,24 +25,96 @@ export declare const officialPlugins: {
         readonly path: "connectors/files/connector.yaml";
         readonly tagPrefix: "files-";
     };
+    readonly gcs: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/gcs/connector.yaml";
+        readonly tagPrefix: "gcs-";
+    };
     readonly github: {
         readonly kind: "connector";
         readonly repository: "ingestron/connectors";
         readonly path: "connectors/github/connector.yaml";
         readonly tagPrefix: "github-";
     };
+    readonly hubspot: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/hubspot/connector.yaml";
+        readonly tagPrefix: "hubspot-";
+    };
+    readonly jira: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/jira/connector.yaml";
+        readonly tagPrefix: "jira-";
+    };
     readonly local: {
         readonly kind: "provider";
         readonly repository: "ingestron/provider-local";
         readonly path: "plugin/provider.yaml";
         readonly tagPrefix: "";
     };
+    readonly mysql: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/mysql/connector.yaml";
+        readonly tagPrefix: "mysql-";
+    };
+    readonly onedrive: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/onedrive/connector.yaml";
+        readonly tagPrefix: "onedrive-";
+    };
+    readonly oracle: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/oracle/connector.yaml";
+        readonly tagPrefix: "oracle-";
+    };
+    readonly postgresql: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/postgresql/connector.yaml";
+        readonly tagPrefix: "postgresql-";
+    };
+    readonly s3: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/s3/connector.yaml";
+        readonly tagPrefix: "s3-";
+    };
+    readonly salesforce: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/salesforce/connector.yaml";
+        readonly tagPrefix: "salesforce-";
+    };
+    readonly sftp: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/sftp/connector.yaml";
+        readonly tagPrefix: "sftp-";
+    };
+    readonly sharepoint: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/sharepoint/connector.yaml";
+        readonly tagPrefix: "sharepoint-";
+    };
     readonly "sql-server": {
         readonly kind: "connector";
         readonly repository: "ingestron/connectors";
         readonly path: "connectors/sql-server/connector.yaml";
         readonly tagPrefix: "sql-server-";
     };
+    readonly stripe: {
+        readonly kind: "connector";
+        readonly repository: "ingestron/connectors";
+        readonly path: "connectors/stripe/connector.yaml";
+        readonly tagPrefix: "stripe-";
+    };
 };
 export declare function officialName(reference: string): {
     name: keyof typeof officialPlugins;

package/build/cli/cli/plugin-resolution.js

@@ -27,24 +27,96 @@ export const officialPlugins = {
         path: "connectors/files/connector.yaml",
         tagPrefix: "files-",
     },
+    gcs: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/gcs/connector.yaml",
+        tagPrefix: "gcs-",
+    },
     github: {
         kind: "connector",
         repository: "ingestron/connectors",
         path: "connectors/github/connector.yaml",
         tagPrefix: "github-",
     },
+    hubspot: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/hubspot/connector.yaml",
+        tagPrefix: "hubspot-",
+    },
+    jira: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/jira/connector.yaml",
+        tagPrefix: "jira-",
+    },
     local: {
         kind: "provider",
         repository: "ingestron/provider-local",
         path: "plugin/provider.yaml",
         tagPrefix: "",
     },
+    mysql: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/mysql/connector.yaml",
+        tagPrefix: "mysql-",
+    },
+    onedrive: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/onedrive/connector.yaml",
+        tagPrefix: "onedrive-",
+    },
+    oracle: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/oracle/connector.yaml",
+        tagPrefix: "oracle-",
+    },
+    postgresql: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/postgresql/connector.yaml",
+        tagPrefix: "postgresql-",
+    },
+    s3: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/s3/connector.yaml",
+        tagPrefix: "s3-",
+    },
+    salesforce: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/salesforce/connector.yaml",
+        tagPrefix: "salesforce-",
+    },
+    sftp: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/sftp/connector.yaml",
+        tagPrefix: "sftp-",
+    },
+    sharepoint: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/sharepoint/connector.yaml",
+        tagPrefix: "sharepoint-",
+    },
     "sql-server": {
         kind: "connector",
         repository: "ingestron/connectors",
         path: "connectors/sql-server/connector.yaml",
         tagPrefix: "sql-server-",
     },
+    stripe: {
+        kind: "connector",
+        repository: "ingestron/connectors",
+        path: "connectors/stripe/connector.yaml",
+        tagPrefix: "stripe-",
+    },
 };
 const catalogueUrl = "https://raw.githubusercontent.com/ingestron/connectors/main/catalogue.json";
 const stable = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;

package/build/cli/cli/workflow-commands.js

@@ -185,6 +185,21 @@ export function workflowCommands(app, host) {
         ...options,
         ...(secretsFile ? { envFile: secretsFile } : {}),
     }));
+    app
+        .command("discover")
+        .description("List every field a flow's source offers and write draft contracts for review")
+        .requiredOption("--flow <id>", "Ingestion flow to discover")
+        .option("--from <file>", "Use a source catalogue or exported metadata rows instead of reading the source")
+        .option("--out <directory>", "Directory for draft contracts", "contracts")
+        .option("--local <configuration>", "Local provider configuration used for native or bridge connections")
+        .option("--accept-keys", "Mark the source's primary keys in the drafts")
+        .option("--python <path>", "Optional Python 3.12 bootstrap override")
+        .option("--secrets-file <file>", "Project-relative dotenv file; only declared secret names are passed")
+        .action(({ secretsFile, acceptKeys, ...options }) => run("discover", {
+        ...options,
+        acceptKeys: !!acceptKeys,
+        ...(secretsFile ? { envFile: secretsFile } : {}),
+    }));
     execution
         .command("status <id>")
         .description("Read the durable local run receipt")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ingestron",
-  "version": "0.17.6",
+  "version": "0.17.8",
   "type": "module",
   "description": "Build repeatable data pipelines from reviewed contracts and versioned standards.",
   "license": "Apache-2.0",
@@ -40,7 +40,7 @@
   },
   "dependencies": {
     "@clack/prompts": "1.7.0",
-    "@ingestron/core": "0.12.12",
+    "@ingestron/core": "0.12.14",
     "@modelcontextprotocol/sdk": "1.30.0",
     "commander": "15.0.0",
     "yaml": "2.9.0"