@jxsuite/connector 0.1.0

package/src/columns.ts

@@ -0,0 +1,292 @@
+/**
+ * Columns — single source of truth for field schema → SQL column mapping and value coercion.
+ *
+ * Table column schemas are plain Jx field schemas (the core field union) plus relationship refs
+ * (specs/relationships.md). This module derives the physical column plan for a table — scalar
+ * columns, reference FK columns, and junction tables for table↔table to-many refs — and the two-way
+ * value coercion between JSON payloads and SQL storage.
+ *
+ * Storage model (v1, uniform across sqlite and postgres):
+ *
+ * - String → text · number → real/double precision · integer → integer
+ * - Boolean → integer 0/1 on sqlite, boolean on postgres
+ * - Array/object → JSON-serialized text
+ * - To-one reference (`{ $ref }`) → text FK column named `<field>_id` storing the target id
+ * - To-many reference to a table (`array` of `$ref` into the same data section) → junction table
+ *   `${sourceTable}_${fieldName}` (relationships.md §3), no column on the source table
+ * - To-many reference to any other section → JSON text column of id strings
+ */
+
+import type { TableDef } from "./types.ts";
+
+export type SqlDialectKind = "sqlite" | "postgres";
+
+/** Parsed relationship-ref target: `#/<sectionKey>/<entryName>`. */
+export interface RefTarget {
+  section: string;
+  name: string;
+}
+
+/** Parse a relationship ref pointer, or null when the string is not one. */
+export function parseRefPointer(pointer: unknown): RefTarget | null {
+  if (typeof pointer !== "string") {
+    return null;
+  }
+  const match = pointer.match(/^#\/([A-Za-z][\w-]*)\/([\w.-]+)$/);
+  if (!match) {
+    return null;
+  }
+  return { name: match[2]!, section: match[1]! };
+}
+
+/** How a column's values are stored and coerced. */
+export type ColumnStorage = "text" | "number" | "integer" | "boolean" | "json";
+
+/** One physical column derived from a schema field. */
+export interface ColumnSpec {
+  /** Schema field name (the wire-payload key). */
+  field: string;
+  /** Physical column name (`<field>_id` for to-one references). */
+  column: string;
+  /** SQL data type passed to the Kysely schema builder. */
+  dataType: string;
+  storage: ColumnStorage;
+  /** Set for reference fields (to-one FK columns and to-many JSON id lists). */
+  ref: RefTarget | null;
+  /** True for to-many reference fields stored as JSON id arrays (non-table targets). */
+  manyRef: boolean;
+}
+
+/** A junction table materializing a table↔table to-many reference (relationships.md §3). */
+export interface JunctionSpec {
+  /** Junction table name: `${sourceTable}_${fieldName}`. */
+  table: string;
+  /** Schema field on the source table this junction materializes. */
+  field: string;
+  sourceTable: string;
+  targetTable: string;
+  /** `${sourceTable}_id`. */
+  sourceColumn: string;
+  /** `${targetTable}_id`, suffixed `_2` when the reference is self-referential. */
+  targetColumn: string;
+  sourceIdType: string;
+  targetIdType: string;
+}
+
+/** The full physical plan for one table. */
+export interface TablePlan {
+  table: string;
+  idType: "uuid" | "integer";
+  timestamps: boolean;
+  columns: ColumnSpec[];
+  junctions: JunctionSpec[];
+  indexes: string[][];
+}
+
+/** SQL data type of a table's primary key. */
+export function idDataType(idKind: "uuid" | "integer"): string {
+  return idKind === "uuid" ? "text" : "integer";
+}
+
+/** Map a scalar field schema type to the SQL data type for a dialect. */
+function scalarDataType(type: string | undefined, dialect: SqlDialectKind): string {
+  switch (type) {
+    case "number": {
+      return dialect === "postgres" ? "double precision" : "real";
+    }
+    case "integer": {
+      return "integer";
+    }
+    case "boolean": {
+      return dialect === "postgres" ? "boolean" : "integer";
+    }
+    default: {
+      return "text";
+    }
+  }
+}
+
+/** Storage class for a scalar field schema type. */
+function scalarStorage(type: string | undefined): ColumnStorage {
+  switch (type) {
+    case "number": {
+      return "number";
+    }
+    case "integer": {
+      return "integer";
+    }
+    case "boolean": {
+      return "boolean";
+    }
+    case "array":
+    case "object": {
+      return "json";
+    }
+    default: {
+      return "text";
+    }
+  }
+}
+
+/**
+ * Derive the physical plan for a table: columns, junction tables, and index column lists.
+ *
+ * @param {string} tableName - The data-section key naming the table
+ * @param {TableDef} table - The table definition
+ * @param {Record<string, TableDef>} allTables - The full data section (junction id types)
+ * @param {SqlDialectKind} dialect - SQL dialect family
+ * @returns {TablePlan}
+ */
+export function planTable(
+  tableName: string,
+  table: TableDef,
+  allTables: Record<string, TableDef>,
+  dialect: SqlDialectKind,
+): TablePlan {
+  const idType = table.id ?? "uuid";
+  const columns: ColumnSpec[] = [];
+  const junctions: JunctionSpec[] = [];
+
+  for (const [field, schema] of Object.entries(table.schema.properties ?? {})) {
+    const toOne = parseRefPointer(schema.$ref);
+    if (toOne) {
+      columns.push({
+        column: `${field}_id`,
+        dataType: "text",
+        field,
+        manyRef: false,
+        ref: toOne,
+        storage: "text",
+      });
+      continue;
+    }
+
+    const itemsRef = schema.type === "array" ? parseRefPointer(schema.items?.$ref) : null;
+    if (itemsRef && itemsRef.section === "data" && allTables[itemsRef.name]) {
+      const target = allTables[itemsRef.name]!;
+      const selfRef = itemsRef.name === tableName;
+      junctions.push({
+        field,
+        sourceColumn: `${tableName}_id`,
+        sourceIdType: idDataType(idType),
+        sourceTable: tableName,
+        table: `${tableName}_${field}`,
+        targetColumn: selfRef ? `${itemsRef.name}_id_2` : `${itemsRef.name}_id`,
+        targetIdType: idDataType(target.id ?? "uuid"),
+        targetTable: itemsRef.name,
+      });
+      continue;
+    }
+    if (itemsRef) {
+      // To-many reference into a non-table section (e.g. content): JSON array of id strings.
+      columns.push({
+        column: field,
+        dataType: "text",
+        field,
+        manyRef: true,
+        ref: itemsRef,
+        storage: "json",
+      });
+      continue;
+    }
+
+    columns.push({
+      column: field,
+      dataType: scalarDataType(schema.type, dialect),
+      field,
+      manyRef: false,
+      ref: null,
+      storage: scalarStorage(schema.type),
+    });
+  }
+
+  // The owner column backs "owner" permission rules even when the schema does not declare it —
+  // The data mount writes it via setColumns and scopes reads/writes through whereOwner.
+  if (table.ownerField && !columns.some((c) => c.field === table.ownerField)) {
+    columns.push({
+      column: table.ownerField,
+      dataType: "text",
+      field: table.ownerField,
+      manyRef: false,
+      ref: null,
+      storage: "text",
+    });
+  }
+
+  const byField = new Map(columns.map((c) => [c.field, c.column]));
+  const indexes: string[][] = [];
+  for (const index of table.indexes ?? []) {
+    const fields = Array.isArray(index) ? index : [index];
+    const cols = fields.map((f) => byField.get(f) ?? f);
+    indexes.push(cols);
+  }
+
+  return {
+    columns,
+    idType,
+    indexes,
+    junctions,
+    table: tableName,
+    timestamps: table.timestamps !== false,
+  };
+}
+
+/** Generate a fresh primary key for an insert, or undefined when the database assigns one. */
+export function newRowId(idKind: "uuid" | "integer"): string | undefined {
+  return idKind === "uuid" ? crypto.randomUUID() : undefined;
+}
+
+/**
+ * Coerce one JSON payload value to its SQL storage representation.
+ *
+ * @param {ColumnSpec} spec
+ * @param {unknown} value - Already-validated payload value
+ * @param {SqlDialectKind} dialect
+ * @returns {unknown}
+ */
+export function toStorage(spec: ColumnSpec, value: unknown, dialect: SqlDialectKind): unknown {
+  if (value === null || value === undefined) {
+    return null;
+  }
+  switch (spec.storage) {
+    case "boolean": {
+      return dialect === "postgres" ? value === true : value === true ? 1 : 0;
+    }
+    case "json": {
+      return JSON.stringify(value);
+    }
+    default: {
+      return value;
+    }
+  }
+}
+
+/**
+ * Coerce a database row back to its JSON payload shape (booleans, JSON columns).
+ *
+ * @param {ColumnSpec[]} specs - Column plan of the table
+ * @param {Record<string, unknown>} row - Raw row from the driver
+ * @returns {Record<string, unknown>}
+ */
+export function fromStorage(
+  specs: ColumnSpec[],
+  row: Record<string, unknown>,
+): Record<string, unknown> {
+  const out: Record<string, unknown> = { ...row };
+  for (const spec of specs) {
+    const value = out[spec.column];
+    if (value === null || value === undefined) {
+      continue;
+    }
+    if (spec.storage === "boolean" && typeof value !== "boolean") {
+      out[spec.column] = value === 1 || value === "1" || value === true;
+    } else if (spec.storage === "json" && typeof value === "string") {
+      try {
+        out[spec.column] = JSON.parse(value) as unknown;
+      } catch {
+        // Malformed stored JSON stays a string rather than failing the read.
+      }
+    }
+  }
+  return out;
+}

package/src/connectors.ts

@@ -0,0 +1,65 @@
+/**
+ * Connectors — the first-party provider map and the `resolveDialect` seam.
+ *
+ * `resolveDialect(connectionName, projectConfig, env)` is the contract dependents build on (the
+ * auth extension resolves its Better Auth database through it): look the connection up in the
+ * project's `connections` section, find its provider, and return the Kysely dialect plus the SQL
+ * dialect family. Hosts with an extension registry should prefer registry dispatch — this map
+ * covers the first-party providers only.
+ */
+
+import { D1 } from "./d1.ts";
+import { Sqlite } from "./sqlite.ts";
+import { Supabase } from "./supabase.ts";
+import type { Dialect } from "kysely";
+import type { SqlDialectKind } from "./columns.ts";
+import type { ConnectorProvider } from "./provider-utils.ts";
+import type { ConnectionDef } from "./types.ts";
+
+export type { ConnectorProvider } from "./provider-utils.ts";
+export { tablesForConnection } from "./provider-utils.ts";
+export { D1 } from "./d1.ts";
+export { Sqlite } from "./sqlite.ts";
+export { Supabase } from "./supabase.ts";
+
+/** First-party providers keyed by their `connector.provider` id. */
+export const PROVIDERS: Record<string, ConnectorProvider> = {
+  d1: D1,
+  sqlite: Sqlite,
+  supabase: Supabase,
+};
+
+/** The `connections`-bearing slice of a project config. */
+export interface ConnectionsConfig {
+  connections?: Record<string, ConnectionDef>;
+  [key: string]: unknown;
+}
+
+/**
+ * Resolve a named connection to its Kysely dialect and SQL dialect family.
+ *
+ * @param {string} connectionName - Key of the project.json `connections` section
+ * @param {ConnectionsConfig} projectConfig - The (partial) project config
+ * @param {Record<string, unknown>} env - Host environment (bindings + env vars)
+ * @returns {Promise<{ dialect: Dialect; type: SqlDialectKind }>}
+ */
+export async function resolveDialect(
+  connectionName: string,
+  projectConfig: ConnectionsConfig,
+  env: Record<string, unknown>,
+): Promise<{ dialect: Dialect; type: SqlDialectKind }> {
+  const connections = projectConfig.connections ?? {};
+  const connection = connections[connectionName];
+  if (!connection) {
+    throw new Error(`Unknown connection "${connectionName}" (no such connections entry)`);
+  }
+  const provider = PROVIDERS[connection.provider];
+  if (!provider) {
+    throw new Error(
+      `Connection "${connectionName}" names unknown provider "${connection.provider}" ` +
+        `(first-party providers: ${Object.keys(PROVIDERS).join(", ")})`,
+    );
+  }
+  const dialect = await provider.dialect({ ...connection, $name: connectionName }, env);
+  return { dialect, type: provider.kind };
+}

package/src/d1.ts

@@ -0,0 +1,64 @@
+/**
+ * D1 provider — Cloudflare D1 connections.
+ *
+ * Inside a deployed Worker the connection's `binding` names a real D1 binding on `env` and queries
+ * go through kysely-d1. Everywhere else (deploy, tests, compiler-timing SSG bakes) the provider
+ * falls back to the D1 HTTP API using `CLOUDFLARE_API_TOKEN` (+ `accountId` from the connection or
+ * `CLOUDFLARE_ACCOUNT_ID`). Connection entries carry identifiers only; the token is always an env
+ * value (specs/extensions.md §13).
+ */
+
+import { D1Dialect } from "kysely-d1";
+import { createD1HttpDialect } from "./dialects/d1-http.ts";
+import { deployWithDialect, testWithDialect } from "./provider-utils.ts";
+import type { D1Database } from "@cloudflare/workers-types";
+import type { Dialect } from "kysely";
+import type { ConnectorProvider } from "./provider-utils.ts";
+import type { ConnectionDef } from "./types.ts";
+
+/** Resolve a D1 dialect for a connection (binding first, HTTP API fallback). */
+function d1Dialect(connection: ConnectionDef, env: Record<string, unknown>): Dialect {
+  const { binding, databaseId, $name = "connection" } = connection;
+  const bound = binding ? env[binding] : undefined;
+  if (bound) {
+    return new D1Dialect({ database: bound as D1Database });
+  }
+  const apiToken = env["CLOUDFLARE_API_TOKEN"];
+  const accountId = connection.accountId ?? env["CLOUDFLARE_ACCOUNT_ID"];
+  if (typeof apiToken === "string" && typeof accountId === "string" && databaseId) {
+    return createD1HttpDialect({ accountId, apiToken, databaseId });
+  }
+  throw new Error(
+    `D1 connection "${$name}" is unreachable: no "${binding ?? "<binding>"}" binding on env, ` +
+      `and the HTTP API needs databaseId + CLOUDFLARE_API_TOKEN + accountId/CLOUDFLARE_ACCOUNT_ID`,
+  );
+}
+
+export const D1 = {
+  bindings: (connection: ConnectionDef) => ({
+    d1_databases: [
+      {
+        binding: connection.binding ?? "DB",
+        ...(connection.databaseId === undefined ? {} : { database_id: connection.databaseId }),
+        database_name: connection.$name ?? connection.binding ?? "DB",
+      },
+    ],
+  }),
+  deploySchema: async (
+    tables: Parameters<ConnectorProvider["deploySchema"]>[0],
+    connection: ConnectionDef,
+    options: { env: Record<string, unknown>; dryRun?: boolean },
+  ) => {
+    const dialect = d1Dialect(connection, options.env);
+    return deployWithDialect(dialect, "sqlite", tables, { dryRun: options.dryRun ?? false });
+  },
+  dialect: d1Dialect,
+  kind: "sqlite",
+  testConnection: async (connection: ConnectionDef, env: Record<string, unknown>) => {
+    try {
+      return await testWithDialect(d1Dialect(connection, env));
+    } catch (error) {
+      return { error: error instanceof Error ? error.message : String(error), ok: false };
+    }
+  },
+} satisfies ConnectorProvider;

package/src/ddl.ts

@@ -0,0 +1,201 @@
+/**
+ * DDL — additive-only schema sync via the Kysely schema builder and introspector.
+ *
+ * The sync creates missing tables (with id and timestamp columns), adds missing columns to existing
+ * tables, materializes junction tables for table↔table to-many references (specs/relationships.md
+ * §3), and creates missing indexes. It NEVER drops or retypes anything: type drift, columns unknown
+ * to the schema, and orphaned junction tables are reported as warnings, not fixed. `dryRun`
+ * compiles the exact statements without executing them — the shape behind `jx db push --dry-run`
+ * and the deploySchema capability.
+ */
+
+import { sql } from "kysely";
+import { idDataType, planTable } from "./columns.ts";
+import type { ColumnDataType, CreateTableBuilder, Kysely } from "kysely";
+import type { SqlDialectKind, TablePlan } from "./columns.ts";
+import type { DynamicDatabase } from "./query.ts";
+import type { DeploySchemaResult, TableDef } from "./types.ts";
+
+export interface SyncOptions {
+  dialect: SqlDialectKind;
+  /** Compile statements without executing them. */
+  dryRun?: boolean;
+}
+
+/** Columns every synced table owns besides the schema fields. */
+const SYSTEM_COLUMNS = new Set(["id", "created_at", "updated_at"]);
+
+interface Executable {
+  compile: () => { sql: string };
+  execute: () => Promise<unknown>;
+}
+
+/**
+ * Synchronize the `data` section's tables into a database, additively.
+ *
+ * @param {Kysely<DynamicDatabase>} db - Kysely instance over the target connection
+ * @param {Record<string, TableDef>} tables - Table definitions (the tables of one connection)
+ * @param {SyncOptions} options
+ * @returns {Promise<DeploySchemaResult>}
+ */
+export async function syncTables(
+  db: Kysely<DynamicDatabase>,
+  tables: Record<string, TableDef>,
+  options: SyncOptions,
+): Promise<DeploySchemaResult> {
+  const { dialect, dryRun = false } = options;
+  const statements: string[] = [];
+  const warnings: string[] = [];
+
+  const plans = Object.entries(tables).map(([name, def]) => planTable(name, def, tables, dialect));
+
+  const introspected = await db.introspection.getTables({ withInternalKyselyTables: false });
+  const existing = new Map(introspected.map((t) => [t.name, t]));
+  const existingIndexes = await indexNames(db, dialect);
+
+  const run = async (builder: Executable): Promise<void> => {
+    statements.push(builder.compile().sql);
+    if (!dryRun) {
+      await builder.execute();
+    }
+  };
+
+  const desiredNames = new Set<string>();
+  for (const plan of plans) {
+    desiredNames.add(plan.table);
+    for (const junction of plan.junctions) {
+      desiredNames.add(junction.table);
+    }
+  }
+
+  for (const plan of plans) {
+    const table = existing.get(plan.table);
+    if (!table) {
+      await run(buildCreateTable(db, plan, dialect));
+    } else {
+      const byName = new Map(table.columns.map((c) => [c.name, c]));
+      for (const col of plan.columns) {
+        const current = byName.get(col.column);
+        if (!current) {
+          await run(
+            db.schema.alterTable(plan.table).addColumn(col.column, col.dataType as ColumnDataType),
+          );
+          continue;
+        }
+        if (current.dataType.toLowerCase() !== col.dataType.toLowerCase()) {
+          warnings.push(
+            `${plan.table}.${col.column}: type drift (database has "${current.dataType}", ` +
+              `schema wants "${col.dataType}") — additive-only sync never retypes`,
+          );
+        }
+      }
+      if (plan.timestamps) {
+        for (const stamp of ["created_at", "updated_at"]) {
+          if (!byName.has(stamp)) {
+            await run(db.schema.alterTable(plan.table).addColumn(stamp, "text"));
+          }
+        }
+      }
+      const declared = new Set(plan.columns.map((c) => c.column));
+      for (const col of table.columns) {
+        if (!declared.has(col.name) && !SYSTEM_COLUMNS.has(col.name)) {
+          warnings.push(
+            `${plan.table}.${col.name}: column exists in the database but not in the schema — ` +
+              `never dropped`,
+          );
+        }
+      }
+    }
+
+    for (const junction of plan.junctions) {
+      if (!existing.has(junction.table)) {
+        await run(
+          db.schema
+            .createTable(junction.table)
+            .addColumn(junction.sourceColumn, junction.sourceIdType as ColumnDataType, (c) =>
+              c.notNull(),
+            )
+            .addColumn(junction.targetColumn, junction.targetIdType as ColumnDataType, (c) =>
+              c.notNull(),
+            )
+            .addPrimaryKeyConstraint(`${junction.table}_pk`, [
+              junction.sourceColumn,
+              junction.targetColumn,
+            ]),
+        );
+      }
+      const indexName = `${junction.table}_${junction.targetColumn}_idx`;
+      if (!existingIndexes.has(indexName)) {
+        await run(
+          db.schema.createIndex(indexName).on(junction.table).columns([junction.targetColumn]),
+        );
+      }
+    }
+
+    for (const columns of plan.indexes) {
+      const indexName = `${plan.table}_${columns.join("_")}_idx`;
+      if (!existingIndexes.has(indexName)) {
+        await run(db.schema.createIndex(indexName).on(plan.table).columns(columns));
+      }
+    }
+  }
+
+  // Orphaned junction tables (a renamed to-many field leaves its old junction behind).
+  const knownTables = new Set(plans.map((p) => p.table));
+  for (const name of existing.keys()) {
+    if (desiredNames.has(name) || knownTables.has(name)) {
+      continue;
+    }
+    const looksLikeJunction = [...knownTables].some((t) => name.startsWith(`${t}_`));
+    if (looksLikeJunction) {
+      warnings.push(
+        `${name}: junction table no longer matches any to-many field — orphaned, never dropped`,
+      );
+    }
+  }
+
+  return { applied: !dryRun, statements, warnings };
+}
+
+/** Build the CREATE TABLE for a plan (id strategy, timestamps, schema columns). */
+function buildCreateTable(
+  db: Kysely<DynamicDatabase>,
+  plan: TablePlan,
+  dialect: SqlDialectKind,
+): CreateTableBuilder<string, string> {
+  let builder = db.schema.createTable(plan.table) as CreateTableBuilder<string, string>;
+  const idType = idDataType(plan.idType) as ColumnDataType;
+  builder =
+    plan.idType === "integer"
+      ? builder.addColumn("id", idType, (c) =>
+          dialect === "postgres"
+            ? c.primaryKey().generatedAlwaysAsIdentity()
+            : c.primaryKey().autoIncrement(),
+        )
+      : builder.addColumn("id", idType, (c) => c.primaryKey().notNull());
+  if (plan.timestamps) {
+    builder = builder.addColumn("created_at", "text").addColumn("updated_at", "text");
+  }
+  for (const col of plan.columns) {
+    builder = builder.addColumn(col.column, col.dataType as ColumnDataType);
+  }
+  return builder;
+}
+
+/** Names of existing indexes (dialect-specific catalog queries; Kysely has no index introspection). */
+async function indexNames(
+  db: Kysely<DynamicDatabase>,
+  dialect: SqlDialectKind,
+): Promise<Set<string>> {
+  try {
+    const query =
+      dialect === "postgres"
+        ? sql<{ name: string }>`select indexname as name from pg_indexes`
+        : sql<{ name: string }>`select name from sqlite_master where type = 'index'`;
+    const { rows } = await query.execute(db);
+    return new Set(rows.map((r) => r.name));
+  } catch {
+    // No catalog access (exotic dialect): fall back to emitting CREATE INDEX unconditionally.
+    return new Set();
+  }
+}