@etohq/workflows-input-dataset 0.0.1-next-20260318155517

package/README.md

@@ -0,0 +1,21 @@
+# @etohq/workflows-input-dataset
+
+Domain-agnostic dataset import spec for turning tabular sources (starting with CSV) into
+`@etohq/workflows-input-schema` submissions.
+
+This package contains types only. Runtime behavior (CSV parsing + mapping) lives in
+`@etohq/workflows-input-dataset-runtime`.
+
+## What This Enables
+
+- Different tenants can import "anything" (admission lists, addresses, tenants, etc.)
+- Mapping from arbitrary CSV headers to schema field keys is programmable via specs, not code.
+
+## Core Type
+
+`DatasetImportSpec`:
+
+- `target.schema_id/schema_version`: which input schema to produce submissions for
+- `mappings[]`: how CSV columns map to schema fields
+- `behavior`: how strict to be on unknown columns and empty strings
+

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./spec";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAA"}

package/dist/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./spec"), exports);
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,yCAAsB"}

package/dist/spec.d.ts

@@ -0,0 +1,187 @@
+import type { InputTransform, InputValueType, ISODateString } from "@etohq/workflows-input-schema";
+/**
+ * Canonical tabular source kinds. We start with CSV because it's the most common onboarding format.
+ */
+export type TabularSourceType = "csv";
+/**
+ * Preprocessing steps applied to extracted text before attempting record extraction.
+ *
+ * Keep this declarative so specs are portable and safe.
+ */
+export type TextPreprocessSpec = {
+    type: "trim";
+} | {
+    type: "normalize_whitespace";
+} | {
+    type: "remove_empty_lines";
+} | {
+    type: "remove_lines_matching";
+    pattern: string;
+    flags?: string | null;
+} | {
+    type: "regex_replace";
+    pattern: string;
+    replace_with: string;
+    flags?: string | null;
+};
+/**
+ * Extracts tabular-ish records from text using regular expressions.
+ *
+ * Recommended: use named capture groups (e.g. `(?<Appnum>\\w+)`) and then map to output columns.
+ */
+export type TextRecordExtractSpec = {
+    type: "line_regex";
+    pattern: string;
+    flags?: string | null;
+    /** If provided, only these group names are emitted (after mapping). */
+    groups?: string[] | null;
+    /** Maps capture-group name -> output column name. */
+    column_map?: Record<string, string> | null;
+} | {
+    type: "global_regex";
+    pattern: string;
+    flags?: string | null;
+    groups?: string[] | null;
+    column_map?: Record<string, string> | null;
+} | {
+    /**
+     * Token-based extraction for fixed/whitespace-aligned PDF rows.
+     *
+     * After preprocessing (especially `normalize_whitespace`), each line is split by spaces.
+     * Column values are selected by token indices (supports negative indices).
+     */
+    type: "token_columns";
+    line_filter?: {
+        pattern: string;
+        flags?: string | null;
+    } | null;
+    columns: Array<{
+        column: string;
+        /** Inclusive token index. Negative indices are from the end (e.g. -1 is last token). */
+        from: number;
+        /** Inclusive token index. If omitted/null, defaults to `from`. */
+        to?: number | null;
+    }>;
+};
+/**
+ * PDF source extraction spec.
+ *
+ * The `pdf-parse` layer only yields plain text. This spec defines how to turn that text into
+ * `Record<string, string>[]` so existing mapping/import logic can run unchanged.
+ */
+export type PdfExtractSpec = {
+    preprocess?: TextPreprocessSpec[] | null;
+    records: TextRecordExtractSpec;
+};
+export type DatasetRefSpec = {
+    id: string;
+    version: number;
+};
+/**
+ * Which input schema this dataset import spec targets.
+ */
+export type DatasetTargetSchemaSpec = {
+    schema_id: string;
+    schema_version: number;
+};
+/**
+ * Controls how the importer behaves when the input doesn't match the spec.
+ */
+export type DatasetImportBehaviorSpec = {
+    on_unknown_columns?: "ignore" | "warn" | "error" | null;
+    empty_string_as_null?: boolean | null;
+    skip_empty_rows?: boolean | null;
+};
+/**
+ * Identifies a column in a tabular source.
+ *
+ * This is intentionally friendly for end-users:
+ * - names can be listed as aliases (e.g. ["Email", "E-mail", "email_address"])
+ * - matching can be case-insensitive
+ */
+export type ColumnSelectorSpec = {
+    type: "by_header";
+    header: string;
+} | {
+    type: "by_header_any";
+    headers: string[];
+    case_insensitive?: boolean | null;
+};
+/**
+ * Parses a cell (string) into an `InputValue` shape that `validateSubmission` can further coerce/validate.
+ *
+ * Notes:
+ * - For richer types (date/enum/array), we encode values as JSON-safe wrappers.
+ * - If you need more power, prefer adding a plugin layer rather than allowing functions in specs.
+ */
+export type CellParseSpec = {
+    type: "as_string";
+} | {
+    type: "as_number";
+} | {
+    type: "as_boolean";
+    true_values?: string[] | null;
+    false_values?: string[] | null;
+} | {
+    type: "as_date";
+    format?: string | null;
+    assume_iso?: boolean | null;
+} | {
+    type: "as_enum";
+} | {
+    type: "as_array";
+    delimiter: string;
+    item: Exclude<CellParseSpec, {
+        type: "as_array";
+    }>;
+} | {
+    type: "as_json";
+};
+export type FieldMappingSpec = {
+    field_key: string;
+    source: ColumnSelectorSpec;
+    parse?: CellParseSpec | null;
+    transforms?: InputTransform[] | null;
+    required?: boolean | null;
+    default_value?: string | null;
+};
+/**
+ * Domain-agnostic import spec for turning a tabular source into `InputSubmission[]`.
+ */
+export type DatasetImportSpec = {
+    id: string;
+    version: number;
+    name: string;
+    description?: string | null;
+    source_type: TabularSourceType;
+    target: DatasetTargetSchemaSpec;
+    behavior?: DatasetImportBehaviorSpec | null;
+    mappings: FieldMappingSpec[];
+    metadata?: Record<string, unknown> | null;
+};
+/**
+ * A normalized, JSON-safe intermediate cell value produced by the importer.
+ *
+ * This is intentionally more limited than `InputValue` so it stays stable for tabular inputs.
+ */
+export type ImportedCellValue = null | string | number | boolean | {
+    type: "date";
+    value: ISODateString;
+} | {
+    type: "enum";
+    value: string;
+} | {
+    type: "array";
+    value: ImportedCellValue[];
+} | {
+    type: "json";
+    value: unknown;
+};
+/**
+ * Used by importers to align to schema field value types without embedding business logic.
+ */
+export type FieldTypeHint = {
+    field_key: string;
+    value_type: InputValueType;
+};
+//# sourceMappingURL=spec.d.ts.map

package/dist/spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spec.d.ts","sourceRoot":"","sources":["../src/spec.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAElG;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,KAAK,CAAA;AAErC;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,sBAAsB,CAAA;CAAE,GAChC;IAAE,IAAI,EAAE,oBAAoB,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GACzE;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAA;AAE3F;;;;GAIG;AACH,MAAM,MAAM,qBAAqB,GAC7B;IACE,IAAI,EAAE,YAAY,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,uEAAuE;IACvE,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IACxB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAA;CAC3C,GACD;IACE,IAAI,EAAE,cAAc,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAA;CAC3C,GACD;IACE;;;;;OAKG;IACH,IAAI,EAAE,eAAe,CAAA;IACrB,WAAW,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAA;IAC/D,OAAO,EAAE,KAAK,CAAC;QACb,MAAM,EAAE,MAAM,CAAA;QACd,wFAAwF;QACxF,IAAI,EAAE,MAAM,CAAA;QACZ,kEAAkE;QAClE,EAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KACnB,CAAC,CAAA;CACH,CAAA;AAEL;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,kBAAkB,EAAE,GAAG,IAAI,CAAA;IACxC,OAAO,EAAE,qBAAqB,CAAA;CAC/B,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;CACvB,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,yBAAyB,GAAG;IACtC,kBAAkB,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAA;IACvD,oBAAoB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAA;IACrC,eAAe,CAAC,EAAE,OAAO,GAAG,IAAI,CAAA;CACjC,CAAA;AAED;;;;;;GAMG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,gBAAgB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAA;CAAE,CAAA;AAEnF;;;;;;GAMG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;CAAE,GACrF;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,UAAU,CAAC,EAAE,OAAO,GAAG,IAAI,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC,aAAa,EAAE;QAAE,IAAI,EAAE,UAAU,CAAA;KAAE,CAAC,CAAA;CAAE,GAC3F;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,CAAA;AAEvB,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,kBAAkB,CAAA;IAC1B,KAAK,CAAC,EAAE,aAAa,GAAG,IAAI,CAAA;IAC5B,UAAU,CAAC,EAAE,cAAc,EAAE,GAAG,IAAI,CAAA;IACpC,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAA;IACzB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAC9B,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,uBAAuB,CAAA;IAC/B,QAAQ,CAAC,EAAE,yBAAyB,GAAG,IAAI,CAAA;IAC3C,QAAQ,EAAE,gBAAgB,EAAE,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;CAC1C,CAAA;AAED;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GACzB,IAAI,GACJ,MAAM,GACN,MAAM,GACN,OAAO,GACP;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,aAAa,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAC/B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,iBAAiB,EAAE,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAA;AAEpC;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,cAAc,CAAA;CAC3B,CAAA"}

package/dist/spec.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=spec.js.map

package/dist/spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"spec.js","sourceRoot":"","sources":["../src/spec.ts"],"names":[],"mappings":""}

package/dist/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"root":["../src/index.ts","../src/spec.ts"],"version":"5.8.3"}

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@etohq/workflows-input-dataset",
+  "version": "0.0.1-next-20260318155517",
+  "description": "Domain-agnostic dataset import spec (CSV/tabular mapping) for workflows input schemas",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@etohq/workflows-input-schema": "0.0.1-next-20260318155517"
+  },
+  "devDependencies": {
+    "@types/node": "22.10.5",
+    "rimraf": "5.0.2",
+    "typescript": "5.8.3"
+  },
+  "scripts": {
+    "build": "rimraf dist && tsc --build",
+    "watch": "tsc --build --watch",
+    "test": "exit 0"
+  }
+}