nucleus-core-ts 0.9.781 → 0.9.783

package/scripts/generate-schema.ts

@@ -72,6 +72,7 @@ interface Config {
   payment?: { enabled?: boolean }
   backup?: { enabled?: boolean }
   secrets?: { enabled?: boolean }
+  integrations?: { enabled?: boolean }
   monitoring?: { enabled?: boolean; persistence?: { enabled?: boolean } }
   domains?: { enabled?: boolean }
   'multi-tenant'?: { enabled?: boolean }
@@ -153,6 +154,7 @@ function getEnabledFeatures(config: Config): Set<string> {
   if (config.domains?.enabled) features.add('domains')
   if (config.backup?.enabled) features.add('backup')
   if (config.secrets?.enabled) features.add('secrets')
+  if (config.integrations?.enabled) features.add('integrations')
   if (config.monitoring?.enabled && config.monitoring?.persistence?.enabled !== false)
     features.add('monitoring')
   const isMultiTenant = config['multi-tenant']?.enabled || config.database?.isMultiTenant
@@ -166,6 +168,38 @@ function isTableEnabled(table: TableDef, enabledFeatures: Set<string>): boolean
   return table.feature_set.every((f) => enabledFeatures.has(f))
 }
 
+/**
+ * Which merged tables end up in the generated schema.
+ *
+ * A config entity is always emitted. A system table is emitted only when its
+ * feature is on.
+ *
+ * ⚠️ The trap this closes: "config entity" used to be inferred as "no system
+ * table has this name", and those stopped being the same thing the moment a new
+ * feature shipped a table under a name an app already used. Adding
+ * `integration_sources` to the built-ins made the generator drop the identically
+ * named table an app had defined for ITSELF, because that app has not switched
+ * the new feature on — and on a deployment with `allowDataLoss` the next schema
+ * push then drops the real table and everything in it.
+ *
+ * Exported for the regression test; the shape is a pure set operation.
+ */
+export function selectEmittedTables<T extends { table_name: string }>(
+  allTables: readonly T[],
+  sets: {
+    systemTableNames: ReadonlySet<string>
+    enabledTableNames: ReadonlySet<string>
+    customTableNames: ReadonlySet<string>
+  }
+): T[] {
+  return allTables.filter(
+    (t) =>
+      sets.customTableNames.has(t.table_name) ||
+      !sets.systemTableNames.has(t.table_name) ||
+      sets.enabledTableNames.has(t.table_name)
+  )
+}
+
 function generateColumnCode(col: Column, allTables: string[], currentTable?: string): string {
   const drizzleType = TYPE_MAP[col.type] || 'text'
   let code = ''
@@ -585,10 +619,11 @@ function main() {
     process.exit(1)
   }
 
-  // Emit config entities always; emit a system table only when its feature is on.
-  const emittedTables = allTables.filter(
-    (t) => !systemTableNames.has(t.table_name) || enabledTableNames.has(t.table_name)
-  )
+  const emittedTables = selectEmittedTables(allTables, {
+    systemTableNames,
+    enabledTableNames,
+    customTableNames: new Set(customTables.map((t) => t.table_name)),
+  })
 
   if (!fs.existsSync(outputDir)) {
     fs.mkdirSync(outputDir, { recursive: true })
@@ -603,4 +638,7 @@ function main() {
   console.log(`Generated: ${path.join(outputDir, 'relations.ts')}`)
 }
 
-main()
+// Only when run as a command. Without this guard, importing anything from this
+// file — a test, or another script reusing a helper — executes the CLI, which
+// exits the process on missing arguments.
+if (import.meta.main) main()

package/src/system.tables.json

@@ -5101,6 +5101,368 @@
         { "name": "last_used_at", "type": "timestamptz" }
       ],
       "indexes": [{ "columns": ["scope", "key"], "unique": true }, { "columns": ["scope"] }]
+    },
+    {
+      "table_name": "integration_sources",
+      "feature_set": ["integrations"],
+      "add_base_columns": true,
+      "is_form_data": false,
+      "available_app_ids": ["default_be"],
+      "available_schemas": ["*"],
+      "excluded_schemas": [],
+      "excluded_methods": [],
+      "bulk_endpoints_enabled": false,
+      "group_name": "Integrations",
+      "columns": [
+        { "name": "name", "type": "varchar", "length": 150, "notNull": true },
+        {
+          "name": "slug",
+          "type": "varchar",
+          "length": 150,
+          "notNull": true,
+          "comment": "Stable machine name; referenced by exports and by operators in logs"
+        },
+        {
+          "name": "base_url",
+          "type": "varchar",
+          "length": 500,
+          "notNull": true,
+          "comment": "Any scheme, host and port — https, plain http, in-cluster DNS, IP. HTTPS is NOT required and must never be enforced: reaching a service inside the same cluster over http is a legitimate deployment."
+        },
+        { "name": "description", "type": "text" },
+        { "name": "enabled", "type": "boolean", "default": true },
+        {
+          "name": "auth_type",
+          "type": "varchar",
+          "length": 20,
+          "notNull": true,
+          "default": "none",
+          "enumValues": ["none", "bearer", "api_key", "basic", "oauth2"]
+        },
+        {
+          "name": "auth_token_encrypted",
+          "type": "text",
+          "sensitive": true,
+          "comment": "Static credential (bearer / api key / basic blob), encrypted at rest"
+        },
+        { "name": "auth_token_hint", "type": "varchar", "length": 24 },
+        { "name": "auth_header_name", "type": "varchar", "length": 80, "default": "Authorization" },
+        {
+          "name": "auth_query_param",
+          "type": "varchar",
+          "length": 80,
+          "comment": "When set, an api_key credential travels as this query parameter instead of a header"
+        },
+        { "name": "extra_headers", "type": "jsonb", "default": {} },
+        { "name": "token_url", "type": "varchar", "length": 500 },
+        {
+          "name": "oauth_grant_type",
+          "type": "varchar",
+          "length": 40,
+          "default": "password",
+          "enumValues": ["password", "client_credentials"]
+        },
+        { "name": "oauth_client_id", "type": "varchar", "length": 200 },
+        { "name": "oauth_client_secret_encrypted", "type": "text", "sensitive": true },
+        { "name": "oauth_client_secret_hint", "type": "varchar", "length": 24 },
+        { "name": "oauth_username", "type": "varchar", "length": 200 },
+        { "name": "oauth_password_encrypted", "type": "text", "sensitive": true },
+        { "name": "oauth_password_hint", "type": "varchar", "length": 24 },
+        { "name": "oauth_scope", "type": "varchar", "length": 300 },
+        {
+          "name": "oauth_client_auth",
+          "type": "varchar",
+          "length": 20,
+          "default": "basic",
+          "enumValues": ["basic", "body"]
+        },
+        {
+          "name": "tls_verify",
+          "type": "boolean",
+          "notNull": true,
+          "default": true,
+          "comment": "Per-source certificate verification. Turning it off applies to THIS source's connections only — never to the process, which is what a NODE_TLS_REJECT_UNAUTHORIZED toggle would do."
+        },
+        { "name": "timeout_ms", "type": "integer", "notNull": true, "default": 30000 },
+        {
+          "name": "spec_url",
+          "type": "varchar",
+          "length": 500,
+          "comment": "Optional API description document (OpenAPI/Swagger/Postman/Bruno). Discovery is a convenience: every endpoint can always be defined by hand instead."
+        },
+        {
+          "name": "spec_format",
+          "type": "varchar",
+          "length": 20,
+          "default": "manual",
+          "enumValues": ["manual", "openapi", "swagger", "postman", "bruno", "har"]
+        }
+      ],
+      "constraints": {
+        "unique": [{ "name": "uniq_integration_source_slug", "columns": ["slug"] }]
+      },
+      "indexes": [{ "columns": ["enabled"] }]
+    },
+    {
+      "table_name": "integration_endpoints",
+      "feature_set": ["integrations"],
+      "add_base_columns": true,
+      "is_form_data": false,
+      "available_app_ids": ["default_be"],
+      "available_schemas": ["*"],
+      "excluded_schemas": [],
+      "excluded_methods": [],
+      "bulk_endpoints_enabled": false,
+      "group_name": "Integrations",
+      "columns": [
+        {
+          "name": "source_id",
+          "type": "uuid",
+          "notNull": true,
+          "references": { "table": "integrationSources", "column": "id", "onDelete": "cascade" }
+        },
+        { "name": "tag", "type": "varchar", "length": 120, "notNull": true },
+        { "name": "name", "type": "varchar", "length": 200 },
+        {
+          "name": "method",
+          "type": "varchar",
+          "length": 10,
+          "notNull": true,
+          "default": "GET",
+          "enumValues": ["GET", "POST", "PUT", "PATCH", "DELETE"]
+        },
+        {
+          "name": "path",
+          "type": "varchar",
+          "length": 500,
+          "notNull": true,
+          "comment": "Appended to the source base URL. May carry {brace} or :colon path tokens, whose values come from param_bindings."
+        },
+        { "name": "description", "type": "text" },
+        { "name": "query_params", "type": "jsonb", "default": {} },
+        { "name": "request_body", "type": "jsonb", "default": {} },
+        {
+          "name": "param_bindings",
+          "type": "jsonb",
+          "default": [],
+          "comment": "Where each path/query parameter comes from: a static value, another endpoint's records (fan-out), or a column of one of our own tables."
+        },
+        {
+          "name": "pagination",
+          "type": "jsonb",
+          "default": {},
+          "comment": "Explicitly chosen paging profile (none/page_number/offset/cursor/link_header). Never guessed — a wrong guess silently imports one page."
+        },
+        { "name": "response_root_path", "type": "varchar", "length": 200 },
+        {
+          "name": "sample_response",
+          "type": "jsonb",
+          "default": {},
+          "comment": "A few real records captured by a live test, so the mapping screen still shows fields when the source is unreachable"
+        },
+        { "name": "enabled", "type": "boolean", "default": true }
+      ],
+      "indexes": [{ "columns": ["source_id"] }, { "columns": ["source_id", "tag"] }]
+    },
+    {
+      "table_name": "integration_mappings",
+      "feature_set": ["integrations"],
+      "add_base_columns": true,
+      "is_form_data": false,
+      "available_app_ids": ["default_be"],
+      "available_schemas": ["*"],
+      "excluded_schemas": [],
+      "excluded_methods": [],
+      "bulk_endpoints_enabled": false,
+      "group_name": "Integrations",
+      "columns": [
+        {
+          "name": "endpoint_id",
+          "type": "uuid",
+          "notNull": true,
+          "references": { "table": "integrationEndpoints", "column": "id", "onDelete": "cascade" }
+        },
+        { "name": "name", "type": "varchar", "length": 200, "notNull": true },
+        {
+          "name": "target_entity",
+          "type": "varchar",
+          "length": 120,
+          "notNull": true,
+          "comment": "snake_case table name this mapping writes into"
+        },
+        { "name": "field_mappings", "type": "jsonb", "notNull": true, "default": [] },
+        { "name": "dedup_source_field", "type": "varchar", "length": 200 },
+        {
+          "name": "dedup_target_field",
+          "type": "varchar",
+          "length": 200,
+          "comment": "Target column that identifies the same record across runs. Without it nothing can be matched and every run appends."
+        },
+        {
+          "name": "write_mode",
+          "type": "varchar",
+          "length": 20,
+          "notNull": true,
+          "default": "upsert",
+          "enumValues": ["upsert", "append", "replace"],
+          "comment": "replace is the destructive wipe-and-rewrite; it is no longer the default and is gated by the completeness check."
+        },
+        {
+          "name": "missing_record_policy",
+          "type": "varchar",
+          "length": 20,
+          "notNull": true,
+          "default": "ignore",
+          "enumValues": ["ignore", "deactivate", "delete"],
+          "comment": "What happens to a row that exists here but no longer in the source. deactivate is the right answer for most HR data and could not be expressed before."
+        },
+        { "name": "missing_record_column", "type": "varchar", "length": 120 },
+        { "name": "missing_record_value", "type": "varchar", "length": 200 },
+        { "name": "schedule_cron", "type": "varchar", "length": 120 },
+        {
+          "name": "schedule_timezone",
+          "type": "varchar",
+          "length": 64,
+          "comment": "IANA zone the cron expression is read in. Without it a nightly job drifts with the container's clock."
+        },
+        { "name": "schedule_enabled", "type": "boolean", "notNull": true, "default": false },
+        { "name": "last_run_at", "type": "timestamptz" },
+        { "name": "enabled", "type": "boolean", "default": true }
+      ],
+      "indexes": [
+        { "columns": ["endpoint_id"] },
+        { "columns": ["target_entity"] },
+        { "columns": ["schedule_enabled"] }
+      ]
+    },
+    {
+      "table_name": "integration_runs",
+      "feature_set": ["integrations"],
+      "add_base_columns": true,
+      "is_form_data": false,
+      "available_app_ids": ["default_be"],
+      "available_schemas": ["*"],
+      "excluded_schemas": [],
+      "excluded_methods": ["POST", "PUT", "PATCH", "DELETE"],
+      "bulk_endpoints_enabled": false,
+      "group_name": "Integrations",
+      "columns": [
+        {
+          "name": "mapping_id",
+          "type": "uuid",
+          "references": { "table": "integrationMappings", "column": "id", "onDelete": "set null" },
+          "comment": "Nullable and NOT cascaded: deleting a mapping must not erase the record that it once rewrote a table."
+        },
+        { "name": "target_entity", "type": "varchar", "length": 120 },
+        {
+          "name": "write_mode",
+          "type": "varchar",
+          "length": 20,
+          "enumValues": ["upsert", "append", "replace"]
+        },
+        {
+          "name": "status",
+          "type": "varchar",
+          "length": 20,
+          "notNull": true,
+          "default": "queued",
+          "enumValues": [
+            "queued",
+            "running",
+            "succeeded",
+            "failed",
+            "cancelled",
+            "reverted"
+          ],
+          "comment": "There is deliberately no 'partial': a run either applied in full or not at all."
+        },
+        {
+          "name": "trigger",
+          "type": "varchar",
+          "length": 20,
+          "notNull": true,
+          "default": "manual",
+          "enumValues": ["manual", "schedule", "api"]
+        },
+        { "name": "total_fetched", "type": "integer", "notNull": true, "default": 0 },
+        {
+          "name": "source_reported_total",
+          "type": "integer",
+          "comment": "What the source said it had. When this and total_fetched disagree, a destructive run refuses to start."
+        },
+        { "name": "created_count", "type": "integer", "notNull": true, "default": 0 },
+        { "name": "updated_count", "type": "integer", "notNull": true, "default": 0 },
+        { "name": "skipped_count", "type": "integer", "notNull": true, "default": 0 },
+        { "name": "deleted_count", "type": "integer", "notNull": true, "default": 0 },
+        { "name": "deactivated_count", "type": "integer", "notNull": true, "default": 0 },
+        { "name": "error_count", "type": "integer", "notNull": true, "default": 0 },
+        { "name": "external_call_count", "type": "integer", "notNull": true, "default": 0 },
+        { "name": "issues", "type": "jsonb", "default": [] },
+        { "name": "errors", "type": "jsonb", "default": [] },
+        {
+          "name": "cancel_requested",
+          "type": "boolean",
+          "notNull": true,
+          "default": false,
+          "comment": "Set by the cancel endpoint; the engine checks it at every batch boundary. A dropped HTTP connection is NOT a cancellation."
+        },
+        { "name": "started_at", "type": "timestamptz" },
+        { "name": "finished_at", "type": "timestamptz" },
+        {
+          "name": "pre_images_expire_at",
+          "type": "timestamptz",
+          "comment": "After this, the run can no longer be reverted and the UI says so rather than offering a button that quietly does nothing."
+        },
+        { "name": "reverted_at", "type": "timestamptz" },
+        { "name": "reverted_by", "type": "uuid", "references": { "table": "users", "column": "id" } },
+        { "name": "reverted_run_id", "type": "uuid" }
+      ],
+      "indexes": [
+        { "columns": ["mapping_id"] },
+        { "columns": ["status"] },
+        { "columns": ["target_entity"] },
+        { "columns": ["created_at"] }
+      ]
+    },
+    {
+      "table_name": "integration_run_items",
+      "feature_set": ["integrations"],
+      "add_base_columns": true,
+      "is_form_data": false,
+      "available_app_ids": ["default_be"],
+      "available_schemas": ["*"],
+      "excluded_schemas": [],
+      "excluded_methods": ["POST", "PUT", "PATCH", "DELETE"],
+      "bulk_endpoints_enabled": false,
+      "group_name": "Integrations",
+      "columns": [
+        {
+          "name": "run_id",
+          "type": "uuid",
+          "notNull": true,
+          "references": { "table": "integrationRuns", "column": "id", "onDelete": "cascade" }
+        },
+        { "name": "target_entity", "type": "varchar", "length": 120, "notNull": true },
+        { "name": "row_id", "type": "uuid" },
+        {
+          "name": "action",
+          "type": "varchar",
+          "length": 20,
+          "notNull": true,
+          "enumValues": ["create", "update", "delete", "deactivate"]
+        },
+        {
+          "name": "before",
+          "type": "jsonb",
+          "comment": "The row as it stood before this run touched it. This is what makes a successful run reversible; the bulk delete path already produced these rows and threw them away."
+        },
+        { "name": "after", "type": "jsonb" }
+      ],
+      "indexes": [
+        { "columns": ["run_id"] },
+        { "columns": ["run_id", "action"] },
+        { "columns": ["target_entity", "row_id"] }
+      ]
     }
   ]
 }