@useairfoil/connector-kit 0.6.0 → 0.7.0

package/dist/connector-app-C6_dZnhE.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"connector-app-C6_dZnhE.mjs","names":["runIngestion","RuntimeConfig.loadTableBindings","RuntimeConfig.engineUri","RuntimeConfig.catalog","catalog"],"sources":["../src/connector-app/check.ts","../src/catalog/retry.ts","../src/catalog/table.ts","../src/connector-app/create-table.ts","../src/connector-app/demo.ts","../src/connector-app/start.ts","../src/connector-app/setup.ts","../src/connector-app/index.ts"],"sourcesContent":["import { Context, Effect, Layer, Tuple } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition, ResourceName } from \"../core/types\";\n\nexport type ResourceCheckResult =\n  | { readonly _tag: \"ok\" }\n  | { readonly _tag: \"error\"; readonly message: string };\n\nexport type CheckOptions<Name extends string> = {\n  readonly resources: ReadonlyArray<Name>;\n};\n\ntype ResourceCheckResults<Name extends PropertyKey> = {\n  readonly [Key in Name]: ResourceCheckResult;\n};\n\nexport type CheckResult<\n  Resources extends ReadonlyArray<ResourceDefinition>,\n  Name extends ResourceName<Resources> = ResourceName<Resources>,\n> = ResourceCheckResults<Name>;\n\ntype CheckFailure = { readonly message: string };\n\nconst ok: ResourceCheckResult = { _tag: \"ok\" };\n\nconst error = (message: string): ResourceCheckResult => ({ _tag: \"error\", message });\n\nconst fromEntries = <Name extends string>(\n  entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,\n): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;\n\nconst resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>\n  fromEntries(names.map((name) => Tuple.make(name, result)));\n\n/**\n * Builds the connector service from its layer and validates only the selected\n * resources. The layer remains open until every check finishes.\n */\nexport function check<\n  Identifier,\n  const Resources extends ReadonlyArray<ResourceDefinition>,\n  E extends CheckFailure,\n  R,\n  const Name extends ResourceName<Resources>,\n>(\n  service: Context.Key<Identifier, ConnectorDefinition<Resources>>,\n  layer: Layer.Layer<Identifier, E, R>,\n  options: CheckOptions<Name>,\n): Effect.Effect<CheckResult<Resources, Name>, never, R> {\n  if (options.resources.length === 0) {\n    return Effect.succeed(fromEntries<Name>([]));\n  }\n\n  return service.pipe(\n    Effect.flatMap((connector) =>\n      Effect.forEach(options.resources, (name) => {\n        const resource = connector.resources.find((candidate) => candidate.name === name);\n        const result = resource\n          ? resource.check.pipe(\n              Effect.match({\n                onFailure: (failure) => error(failure.message),\n                onSuccess: () => ok,\n              }),\n            )\n          : Effect.succeed(error(`Unknown connector resource: ${name}`));\n\n        return result.pipe(Effect.map((result) => Tuple.make(name, result)));\n      }).pipe(Effect.map(fromEntries)),\n    ),\n    Effect.provide(layer),\n    Effect.match({\n      onFailure: (failure) => resultsFor(options.resources, error(failure.message)),\n      onSuccess: (results) => results,\n    }),\n  );\n}\n","import { Effect, Schedule, type Duration } from \"effect\";\n\nexport type TableRetryOptions = {\n  readonly interval: Duration.Input;\n  /** Retries after the initial attempt. */\n  readonly times: number;\n};\n\n/** Retries transient catalog responses at the request boundary. */\nexport const retryCatalogRequest = <A, E extends { readonly status?: number }, R>(\n  request: Effect.Effect<A, E, R>,\n  options?: TableRetryOptions,\n) =>\n  options === undefined\n    ? request\n    : request.pipe(\n        Effect.retry({\n          schedule: Schedule.spaced(options.interval),\n          times: options.times,\n          while: (error) =>\n            error.status === 408 ||\n            error.status === 429 ||\n            (error.status !== undefined && error.status >= 500 && error.status <= 599),\n        }),\n      );\n","import type {\n  IcebergType,\n  StructField,\n  TableIdentifier,\n  TableMetadata,\n  TableSchema,\n} from \"iceberg-js\";\n\nimport { CatalogManager } from \"@useairfoil/api\";\nimport { Effect, type Schema } from \"effect\";\nimport { HttpClient } from \"effect/http\";\nimport { getCurrentSchema } from \"iceberg-js\";\n\nimport { ConnectorError } from \"../errors\";\nimport { IcebergSchemaError } from \"../iceberg/error\";\nimport { compileTable, makeCreateRequest } from \"../iceberg/table\";\nimport { retryCatalogRequest, type TableRetryOptions } from \"./retry\";\n\ntype Catalog = Effect.Success<\n  ReturnType<CatalogManager.CatalogManagerService[\"getIcebergCatalog\"]>\n>;\n\ntype TableOptions = {\n  readonly retry?: TableRetryOptions;\n  readonly catalog: Pick<Catalog, \"loadTable\" | \"createTable\" | \"createNamespaceIfNotExists\">;\n  readonly identifier: TableIdentifier;\n  readonly location?: string | undefined;\n  readonly properties?: Readonly<Record<string, string>> | undefined;\n};\n\ntype EngineTableOptions = {\n  readonly engineUri: string;\n  readonly catalogId: string;\n  readonly identifier: TableIdentifier;\n  readonly location?: string | undefined;\n  readonly properties?: Readonly<Record<string, string>> | undefined;\n};\n\nconst tableError = (cause: unknown): ConnectorError => {\n  if (cause instanceof ConnectorError) return cause;\n\n  const message = cause instanceof Error ? cause.message : \"Failed to prepare Iceberg table\";\n  return new ConnectorError({\n    message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,\n    cause,\n  });\n};\n\nconst compareTypes = (\n  expected: IcebergType,\n  actual: IcebergType,\n  path: string,\n): string | undefined => {\n  if (typeof expected === \"string\" || typeof actual === \"string\") {\n    return expected === actual ? undefined : path;\n  }\n\n  if (expected.type !== actual.type) return path;\n\n  switch (expected.type) {\n    case \"struct\":\n      if (actual.type !== \"struct\") return path;\n      return compareFields(expected.fields, actual.fields, path);\n    case \"list\":\n      if (actual.type !== \"list\" || expected[\"element-required\"] !== actual[\"element-required\"]) {\n        return path;\n      }\n      return compareTypes(expected.element, actual.element, `${path}.element`);\n    case \"map\":\n      if (actual.type !== \"map\" || expected[\"value-required\"] !== actual[\"value-required\"]) {\n        return path;\n      }\n      return (\n        compareTypes(expected.key, actual.key, `${path}.key`) ??\n        compareTypes(expected.value, actual.value, `${path}.value`)\n      );\n  }\n};\n\nconst compareFields = (\n  expected: ReadonlyArray<StructField>,\n  actual: ReadonlyArray<StructField>,\n  path: string,\n): string | undefined => {\n  if (expected.length !== actual.length) return path;\n\n  for (const field of expected) {\n    const fieldPath = `${path}.${field.name}`;\n    const other = actual.find((candidate) => candidate.name === field.name);\n\n    if (!other || field.required !== other.required) return fieldPath;\n\n    const mismatch = compareTypes(field.type, other.type, fieldPath);\n    if (mismatch) return mismatch;\n  }\n};\n\nconst verifyTable = (\n  metadata: TableMetadata,\n  expected: TableSchema,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      if (!Array.isArray(metadata.schemas)) throw new Error(\"Invalid table metadata\");\n      if (metadata[\"format-version\"] !== 3) {\n        throw new Error(\n          \"Storage-engine tables must use Iceberg format v3. Migrate the table before ingesting.\",\n        );\n      }\n      const partitionSpec = metadata[\"partition-specs\"].find(\n        (spec) => spec[\"spec-id\"] === metadata[\"default-spec-id\"],\n      );\n      if (!partitionSpec) throw new Error(\"Table has no default partition spec\");\n      if (partitionSpec.fields.length > 0) {\n        throw new Error(\"Storage-engine tables must be unpartitioned.\");\n      }\n      if (metadata.properties?.[\"encryption.key-id\"] !== undefined) {\n        throw new Error(\"Storage-engine tables must be unencrypted.\");\n      }\n      const actual = getCurrentSchema(metadata);\n      if (!actual) throw new Error(\"Table has no current schema\");\n\n      const path = compareFields(expected.fields, actual.fields, \"$\");\n      if (path) {\n        throw new Error(\n          `Table schema does not match at ${path}. Migrate the table or revert the schema change.`,\n        );\n      }\n\n      const identifierNames = (schema: TableSchema) =>\n        new Set(\n          schema.fields\n            .filter((field) => schema[\"identifier-field-ids\"]?.includes(field.id))\n            .map((field) => field.name),\n        );\n      const expectedNames = identifierNames(expected);\n      const actualNames = identifierNames(actual);\n      if (\n        expectedNames.size !== actualNames.size ||\n        [...expectedNames].some((name) => !actualNames.has(name))\n      ) {\n        throw new Error(\n          \"Table identifier fields do not match. Recreate the table or revert the schema change.\",\n        );\n      }\n\n      return actual;\n    },\n    catch: (cause) =>\n      new ConnectorError({\n        message: cause instanceof Error ? cause.message : \"Invalid table metadata\",\n        cause,\n      }),\n  });\n\n/** Creates a table through the REST catalog, then checks the loaded schema. */\nexport const createTable = (\n  schema: Schema.Top,\n  options: TableOptions,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.gen(function* () {\n    const table = yield* compileTable(schema);\n    yield* options.catalog.createTable(\n      { namespace: options.identifier.namespace },\n      makeCreateRequest(table, {\n        name: options.identifier.name,\n        location: options.location,\n        properties: options.properties,\n      }),\n    );\n\n    const metadata = yield* options.catalog.loadTable(options.identifier);\n    return yield* verifyTable(metadata, table.schema);\n  }).pipe(Effect.mapError(tableError));\n\n/** Creates a missing table through an engine catalog and verifies its schema. */\nexport const ensureTableFromEngine = (\n  schema: Schema.Top,\n  options: EngineTableOptions,\n): Effect.Effect<TableSchema, ConnectorError, HttpClient.HttpClient> =>\n  Effect.scoped(\n    Effect.gen(function* () {\n      const manager = yield* CatalogManager.make({ engineUri: options.engineUri });\n      const catalog = yield* manager.getIcebergCatalog(options.catalogId);\n      return yield* ensureTable(schema, {\n        catalog,\n        identifier: options.identifier,\n        location: options.location,\n        properties: options.properties,\n      });\n    }),\n  ).pipe(Effect.mapError(tableError));\n\n/** Loads a table, creating it when it is missing. */\nexport const ensureTable = (\n  schema: Schema.Top,\n  options: TableOptions,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.gen(function* () {\n    const table = yield* compileTable(schema);\n    const request = <A>(\n      effect: Effect.Effect<A, Effect.Error<ReturnType<Catalog[\"loadConfig\"]>>>,\n    ) => retryCatalogRequest(effect, options.retry);\n    const metadata = yield* request(options.catalog.loadTable(options.identifier)).pipe(\n      Effect.catchIf(\n        (error) => error.status === 404,\n        () =>\n          request(\n            options.catalog.createNamespaceIfNotExists({ namespace: options.identifier.namespace }),\n          )\n            .pipe(\n              Effect.andThen(\n                request(\n                  options.catalog.createTable(\n                    { namespace: options.identifier.namespace },\n                    makeCreateRequest(table, {\n                      name: options.identifier.name,\n                      location: options.location,\n                      properties: options.properties,\n                    }),\n                  ),\n                ),\n              ),\n            )\n            .pipe(\n              // Another instance may have created the table. Reload and check it.\n              Effect.catchIf(\n                (error) => error.status === 409 || error.status === 412,\n                () => Effect.void,\n              ),\n              Effect.flatMap(() => request(options.catalog.loadTable(options.identifier))),\n            ),\n      ),\n    );\n\n    return yield* verifyTable(metadata, table.schema);\n  }).pipe(Effect.mapError(tableError));\n","import { Effect, Option, type Schema } from \"effect\";\nimport { Argument, CliError, Command, Flag } from \"effect/cli\";\n\nimport { ensureTableFromEngine } from \"../catalog/table\";\nimport { engineUri } from \"../runtime-config/config\";\n\n/** Creates the table command; existing tables must match the connector's row schemas. */\nexport const makeCreateTableCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>\n  Command.make(\n    \"create-table\",\n    {\n      table: Argument.String(\"table\"),\n      catalog: Flag.String(\"catalog\"),\n      location: Flag.String(\"location\").pipe(Flag.optional),\n      resource: Flag.String(\"resource\").pipe(Flag.optional),\n    },\n    ({ table, catalog, location, resource }) =>\n      Effect.gen(function* () {\n        const namespace = table.split(\".\");\n        const name = namespace.pop();\n\n        if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) {\n          return yield* new CliError.UserError({\n            cause: \"Invalid table name\",\n            userMessage: \"Use a table name like my.namespace.table\",\n          });\n        }\n\n        const resourceName = Option.getOrElse(resource, () => name);\n        const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : undefined;\n        if (!schema) {\n          return yield* new CliError.UserError({\n            cause: \"Unknown resource\",\n            userMessage: `Unknown resource: ${resourceName}`,\n          });\n        }\n\n        const uri = yield* engineUri;\n        yield* ensureTableFromEngine(schema, {\n          engineUri: uri.href,\n          catalogId: catalog,\n          identifier: { namespace, name },\n          location: Option.getOrUndefined(location),\n        });\n        yield* Effect.log(`Prepared ${table}`);\n      }),\n  ).pipe(Command.withDescription(\"Create or verify an Iceberg table for a connector resource\"));\n","import { DateTime, Effect, Option, Schema } from \"effect\";\nimport { Command, Flag } from \"effect/cli\";\nimport humanFormat from \"human-format\";\n\nimport { run, type DemoDefinition } from \"../demo\";\nimport { layerArrowFlightConfig } from \"../ingestor/arrow-flight\";\n\n/** Reads byte sizes such as `512kB`, `128MB` or `1.5GB`. */\nexport const parseSize = (input: string): Option.Option<number> =>\n  Option.liftThrowable(() => humanFormat.parse.bytes(input))().pipe(\n    Option.filter((bytes) => Number.isSafeInteger(bytes) && bytes > 0),\n  );\n\n/**\n * Creates the command that writes demo data over Arrow Flight, then exits.\n * It reads hosted config and table bindings, so provide `RuntimeConfig.layerHosted()`.\n */\nexport const makeDemoCommand = (demo: DemoDefinition) =>\n  Command.make(\n    \"demo\",\n    {\n      size: Flag.String(\"size\").pipe(\n        Flag.filterMap(parseSize, () => \"a size like 512kB, 128MB or 1.5GB\"),\n      ),\n      seed: Flag.Int(\"seed\").pipe(Flag.optional),\n      batchSize: Flag.Int(\"batch-size\").pipe(\n        Flag.withSchema(Schema.Int.check(Schema.isGreaterThan(0))),\n        Flag.withDescription(\"Maximum rows per ingestion batch (default: 500)\"),\n        Flag.optional,\n      ),\n    },\n    ({ size, seed, batchSize }) =>\n      Effect.gen(function* () {\n        const now = yield* DateTime.nowAsDate;\n\n        yield* run(demo, {\n          bytes: size,\n          seed: Option.getOrUndefined(seed),\n          batchSize: Option.getOrUndefined(batchSize),\n          now,\n        }).pipe(\n          Effect.provide(\n            layerArrowFlightConfig({ resources: [...demo.setup.tables, ...demo.group.tables] }),\n          ),\n        );\n      }),\n  ).pipe(Command.withDescription(\"Write demo data for the bound tables, then exit\"));\n","import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n  readonly port: number;\n  readonly initialCutoff?: Cursor.Value;\n  readonly healthPath?: HttpRouter.PathInput;\n  readonly metricsPath?: HttpRouter.PathInput | false;\n  readonly statusPath?: HttpRouter.PathInput | false;\n  readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n  Effect.gen(function* () {\n    const initialCutoff =\n      options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n    const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n    yield* Effect.logInfo(\"webhook server ready\").pipe(\n      Effect.annotateLogs({ port: options.port, routes }),\n    );\n\n    return yield* runIngestion(app, {\n      initialCutoff,\n      webhook: {\n        routes: app.webhooks ?? [],\n        healthPath: options.healthPath ?? \"/health\",\n        metricsPath: options.metricsPath,\n        statusPath: options.statusPath,\n        disableHttpLogger: options.disableHttpLogger ?? true,\n      },\n    }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n  });\n","import { CatalogManager } from \"@useairfoil/api\";\nimport { Duration, Effect, Option, Schema } from \"effect\";\nimport { Command, Flag } from \"effect/cli\";\n\nimport { retryCatalogRequest } from \"../catalog/retry\";\nimport { ensureTable } from \"../catalog/table\";\nimport * as RuntimeConfig from \"../runtime-config\";\n\n/** Creates or verifies all bound tables without loading provider credentials. */\nexport const makeSetupCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>\n  Command.make(\n    \"setup\",\n    {\n      checkpointIntervalSec: Flag.Int(\"checkpoint-interval-sec\").pipe(\n        Flag.withSchema(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 900 }))),\n        Flag.withDescription(\n          \"Checkpoint interval for new tables in seconds (0 disables the interval delay)\",\n        ),\n        Flag.optional,\n      ),\n      retryInterval: Flag.String(\"retry-interval\").pipe(\n        Flag.withDefault(\"5 seconds\"),\n        Flag.withSchema(Schema.DurationFromString),\n        Flag.filter(\n          (duration) => Duration.isPositive(duration) && Duration.isFinite(duration),\n          () => \"a positive duration, such as 5 seconds\",\n        ),\n        Flag.withDescription(\"Delay between transient catalog retries\"),\n      ),\n      retries: Flag.Int(\"retries\").pipe(\n        Flag.withDefault(10),\n        Flag.filter(\n          (value) => value >= 0,\n          () => \"a non-negative number of retries\",\n        ),\n        Flag.withDescription(\"Maximum retries per catalog request after the initial attempt\"),\n      ),\n    },\n    ({ retryInterval, retries, checkpointIntervalSec }) =>\n      Effect.gen(function* () {\n        const source = {\n          resources: Object.entries(schemas).map(([name, rowSchema]) => ({ name, rowSchema })),\n        };\n        const tables = yield* RuntimeConfig.loadTableBindings(source);\n        const uri = yield* RuntimeConfig.engineUri;\n        const catalogId = yield* RuntimeConfig.catalog;\n        const manager = yield* CatalogManager.make({ engineUri: uri.href });\n        const retry = { interval: retryInterval, times: retries };\n        const properties = Option.match(checkpointIntervalSec, {\n          onNone: () => undefined,\n          onSome: (seconds) => ({ \"airfoil.checkpoint.interval-sec\": String(seconds) }),\n        });\n        const catalog = yield* retryCatalogRequest(manager.getIcebergCatalog(catalogId), retry);\n\n        // Sequential setup avoids bursting the catalog's table-creation quota.\n        for (const resource of source.resources) {\n          const binding = tables[resource.name];\n          if (!binding) continue;\n          yield* ensureTable(resource.rowSchema, {\n            catalog,\n            identifier: binding,\n            location: binding.location,\n            retry,\n            properties,\n          });\n          yield* Effect.logInfo(`Prepared ${[...binding.namespace, binding.name].join(\".\")}`);\n        }\n      }),\n  ).pipe(Command.withDescription(\"Create or verify the Iceberg tables in AIRFOIL_TABLE_BINDINGS\"));\n","export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from \"./check\";\nexport { makeCreateTableCommand } from \"./create-table\";\nexport { makeDemoCommand, parseSize } from \"./demo\";\nexport { start, type App, type StartOptions } from \"./start\";\nexport { makeSetupCommand } from \"./setup\";\n"],"mappings":";;;;;;;;;;;;;;;;AAuBA,MAAM,KAA0B,EAAE,MAAM,KAAK;AAE7C,MAAM,SAAS,aAA0C;CAAE,MAAM;CAAS;AAAQ;AAElF,MAAM,eACJ,YAC+B,OAAO,YAAY,OAAO;AAE3D,MAAM,cAAmC,OAA4B,WACnE,YAAY,MAAM,KAAK,SAAS,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;;;;;AAM3D,SAAgB,MAOd,SACA,OACA,SACuD;CACvD,IAAI,QAAQ,UAAU,WAAW,GAC/B,OAAO,OAAO,QAAQ,YAAkB,CAAC,CAAC,CAAC;CAG7C,OAAO,QAAQ,KACb,OAAO,SAAS,cACd,OAAO,QAAQ,QAAQ,YAAY,SAAS;EAC1C,MAAM,WAAW,UAAU,UAAU,MAAM,cAAc,UAAU,SAAS,IAAI;EAUhF,QATe,WACX,SAAS,MAAM,KACb,OAAO,MAAM;GACX,YAAY,YAAY,MAAM,QAAQ,OAAO;GAC7C,iBAAiB;EACnB,CAAC,CACH,IACA,OAAO,QAAQ,MAAM,+BAA+B,MAAM,CAAC,EAAA,CAEjD,KAAK,OAAO,KAAK,WAAW,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;CACrE,CAAC,CAAC,CAAC,KAAK,OAAO,IAAI,WAAW,CAAC,CACjC,GACA,OAAO,QAAQ,KAAK,GACpB,OAAO,MAAM;EACX,YAAY,YAAY,WAAW,QAAQ,WAAW,MAAM,QAAQ,OAAO,CAAC;EAC5E,YAAY,YAAY;CAC1B,CAAC,CACH;AACF;;;;AClEA,MAAa,uBACX,SACA,YAEA,YAAY,KAAA,IACR,UACA,QAAQ,KACN,OAAO,MAAM;CACX,UAAU,SAAS,OAAO,QAAQ,QAAQ;CAC1C,OAAO,QAAQ;CACf,QAAQ,UACN,MAAM,WAAW,OACjB,MAAM,WAAW,OAChB,MAAM,WAAW,KAAA,KAAa,MAAM,UAAU,OAAO,MAAM,UAAU;AAC1E,CAAC,CACH;;;ACcN,MAAM,cAAc,UAAmC;CACrD,IAAI,iBAAiB,gBAAgB,OAAO;CAE5C,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;CACzD,OAAO,IAAI,eAAe;EACxB,SAAS,iBAAiB,qBAAqB,GAAG,MAAM,KAAK,IAAI,YAAY;EAC7E;CACF,CAAC;AACH;AAEA,MAAM,gBACJ,UACA,QACA,SACuB;CACvB,IAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UACpD,OAAO,aAAa,SAAS,KAAA,IAAY;CAG3C,IAAI,SAAS,SAAS,OAAO,MAAM,OAAO;CAE1C,QAAQ,SAAS,MAAjB;EACE,KAAK;GACH,IAAI,OAAO,SAAS,UAAU,OAAO;GACrC,OAAO,cAAc,SAAS,QAAQ,OAAO,QAAQ,IAAI;EAC3D,KAAK;GACH,IAAI,OAAO,SAAS,UAAU,SAAS,wBAAwB,OAAO,qBACpE,OAAO;GAET,OAAO,aAAa,SAAS,SAAS,OAAO,SAAS,GAAG,KAAK,SAAS;EACzE,KAAK;GACH,IAAI,OAAO,SAAS,SAAS,SAAS,sBAAsB,OAAO,mBACjE,OAAO;GAET,OACE,aAAa,SAAS,KAAK,OAAO,KAAK,GAAG,KAAK,KAAK,KACpD,aAAa,SAAS,OAAO,OAAO,OAAO,GAAG,KAAK,OAAO;CAEhE;AACF;AAEA,MAAM,iBACJ,UACA,QACA,SACuB;CACvB,IAAI,SAAS,WAAW,OAAO,QAAQ,OAAO;CAE9C,KAAK,MAAM,SAAS,UAAU;EAC5B,MAAM,YAAY,GAAG,KAAK,GAAG,MAAM;EACnC,MAAM,QAAQ,OAAO,MAAM,cAAc,UAAU,SAAS,MAAM,IAAI;EAEtE,IAAI,CAAC,SAAS,MAAM,aAAa,MAAM,UAAU,OAAO;EAExD,MAAM,WAAW,aAAa,MAAM,MAAM,MAAM,MAAM,SAAS;EAC/D,IAAI,UAAU,OAAO;CACvB;AACF;AAEA,MAAM,eACJ,UACA,aAEA,OAAO,IAAI;CACT,WAAW;EACT,IAAI,CAAC,MAAM,QAAQ,SAAS,OAAO,GAAG,MAAM,IAAI,MAAM,wBAAwB;EAC9E,IAAI,SAAS,sBAAsB,GACjC,MAAM,IAAI,MACR,uFACF;EAEF,MAAM,gBAAgB,SAAS,kBAAkB,CAAC,MAC/C,SAAS,KAAK,eAAe,SAAS,kBACzC;EACA,IAAI,CAAC,eAAe,MAAM,IAAI,MAAM,qCAAqC;EACzE,IAAI,cAAc,OAAO,SAAS,GAChC,MAAM,IAAI,MAAM,8CAA8C;EAEhE,IAAI,SAAS,aAAa,yBAAyB,KAAA,GACjD,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,SAAS,iBAAiB,QAAQ;EACxC,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,6BAA6B;EAE1D,MAAM,OAAO,cAAc,SAAS,QAAQ,OAAO,QAAQ,GAAG;EAC9D,IAAI,MACF,MAAM,IAAI,MACR,kCAAkC,KAAK,iDACzC;EAGF,MAAM,mBAAmB,WACvB,IAAI,IACF,OAAO,OACJ,QAAQ,UAAU,OAAO,uBAAuB,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CACrE,KAAK,UAAU,MAAM,IAAI,CAC9B;EACF,MAAM,gBAAgB,gBAAgB,QAAQ;EAC9C,MAAM,cAAc,gBAAgB,MAAM;EAC1C,IACE,cAAc,SAAS,YAAY,QACnC,CAAC,GAAG,aAAa,CAAC,CAAC,MAAM,SAAS,CAAC,YAAY,IAAI,IAAI,CAAC,GAExD,MAAM,IAAI,MACR,uFACF;EAGF,OAAO;CACT;CACA,QAAQ,UACN,IAAI,eAAe;EACjB,SAAS,iBAAiB,QAAQ,MAAM,UAAU;EAClD;CACF,CAAC;AACL,CAAC;;AAuBH,MAAa,yBACX,QACA,YAEA,OAAO,OACL,OAAO,IAAI,aAAa;CAEtB,MAAM,UAAU,QAAO,OADA,eAAe,KAAK,EAAE,WAAW,QAAQ,UAAU,CAAC,EAAA,CAC5C,kBAAkB,QAAQ,SAAS;CAClE,OAAO,OAAO,YAAY,QAAQ;EAChC;EACA,YAAY,QAAQ;EACpB,UAAU,QAAQ;EAClB,YAAY,QAAQ;CACtB,CAAC;AACH,CAAC,CACH,CAAC,CAAC,KAAK,OAAO,SAAS,UAAU,CAAC;;AAGpC,MAAa,eACX,QACA,YAEA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,aAAa,MAAM;CACxC,MAAM,WACJ,WACG,oBAAoB,QAAQ,QAAQ,KAAK;CAC9C,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,UAAU,QAAQ,UAAU,CAAC,CAAC,CAAC,KAC7E,OAAO,SACJ,UAAU,MAAM,WAAW,WAE1B,QACE,QAAQ,QAAQ,2BAA2B,EAAE,WAAW,QAAQ,WAAW,UAAU,CAAC,CACxF,CAAC,CACE,KACC,OAAO,QACL,QACE,QAAQ,QAAQ,YACd,EAAE,WAAW,QAAQ,WAAW,UAAU,GAC1C,kBAAkB,OAAO;EACvB,MAAM,QAAQ,WAAW;EACzB,UAAU,QAAQ;EAClB,YAAY,QAAQ;CACtB,CAAC,CACH,CACF,CACF,CACF,CAAC,CACA,KAEC,OAAO,SACJ,UAAU,MAAM,WAAW,OAAO,MAAM,WAAW,WAC9C,OAAO,IACf,GACA,OAAO,cAAc,QAAQ,QAAQ,QAAQ,UAAU,QAAQ,UAAU,CAAC,CAAC,CAC7E,CACN,CACF;CAEA,OAAO,OAAO,YAAY,UAAU,MAAM,MAAM;AAClD,CAAC,CAAC,CAAC,KAAK,OAAO,SAAS,UAAU,CAAC;;;;ACrOrC,MAAa,0BAA0B,YACrC,QAAQ,KACN,gBACA;CACE,OAAO,SAAS,OAAO,OAAO;CAC9B,SAAS,KAAK,OAAO,SAAS;CAC9B,UAAU,KAAK,OAAO,UAAU,CAAC,CAAC,KAAK,KAAK,QAAQ;CACpD,UAAU,KAAK,OAAO,UAAU,CAAC,CAAC,KAAK,KAAK,QAAQ;AACtD,IACC,EAAE,OAAO,SAAS,UAAU,eAC3B,OAAO,IAAI,aAAa;CACtB,MAAM,YAAY,MAAM,MAAM,GAAG;CACjC,MAAM,OAAO,UAAU,IAAI;CAE3B,IAAI,CAAC,QAAQ,UAAU,WAAW,KAAK,UAAU,MAAM,SAAS,KAAK,WAAW,CAAC,GAC/E,OAAO,OAAO,IAAI,SAAS,UAAU;EACnC,OAAO;EACP,aAAa;CACf,CAAC;CAGH,MAAM,eAAe,OAAO,UAAU,gBAAgB,IAAI;CAC1D,MAAM,SAAS,OAAO,OAAO,SAAS,YAAY,IAAI,QAAQ,gBAAgB,KAAA;CAC9E,IAAI,CAAC,QACH,OAAO,OAAO,IAAI,SAAS,UAAU;EACnC,OAAO;EACP,aAAa,qBAAqB;CACpC,CAAC;CAGH,MAAM,MAAM,OAAO;CACnB,OAAO,sBAAsB,QAAQ;EACnC,WAAW,IAAI;EACf,WAAW;EACX,YAAY;GAAE;GAAW;EAAK;EAC9B,UAAU,OAAO,eAAe,QAAQ;CAC1C,CAAC;CACD,OAAO,OAAO,IAAI,YAAY,OAAO;AACvC,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,4DAA4D,CAAC;;;;ACtC9F,MAAa,aAAa,UACxB,OAAO,oBAAoB,YAAY,MAAM,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAC3D,OAAO,QAAQ,UAAU,OAAO,cAAc,KAAK,KAAK,QAAQ,CAAC,CACnE;;;;;AAMF,MAAa,mBAAmB,SAC9B,QAAQ,KACN,QACA;CACE,MAAM,KAAK,OAAO,MAAM,CAAC,CAAC,KACxB,KAAK,UAAU,iBAAiB,mCAAmC,CACrE;CACA,MAAM,KAAK,IAAI,MAAM,CAAC,CAAC,KAAK,KAAK,QAAQ;CACzC,WAAW,KAAK,IAAI,YAAY,CAAC,CAAC,KAChC,KAAK,WAAW,OAAO,IAAI,MAAM,OAAO,cAAc,CAAC,CAAC,CAAC,GACzD,KAAK,gBAAgB,iDAAiD,GACtE,KAAK,QACP;AACF,IACC,EAAE,MAAM,MAAM,gBACb,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,SAAS;CAE5B,OAAO,IAAI,MAAM;EACf,OAAO;EACP,MAAM,OAAO,eAAe,IAAI;EAChC,WAAW,OAAO,eAAe,SAAS;EAC1C;CACF,CAAC,CAAC,CAAC,KACD,OAAO,QACL,uBAAuB,EAAE,WAAW,CAAC,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,MAAM,MAAM,EAAE,CAAC,CACpF,CACF;AACF,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,iDAAiD,CAAC;;;ACzBnF,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;CACnF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,IAAI,KAAK,CAAC;CAE5D,OAAO,OAAO,QAAQ,sBAAsB,CAAC,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;CAAO,CAAC,CACpD;CAEA,OAAO,OAAOA,MAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,CAAC;GACzB,YAAY,QAAQ,cAAc;GAClC,aAAa,QAAQ;GACrB,YAAY,QAAQ;GACpB,mBAAmB,QAAQ,qBAAqB;EAClD;CACF,CAAC,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,KAAK,CAAC,CAAC,CAAC;AACpF,CAAC;;;;AChCH,MAAa,oBAAoB,YAC/B,QAAQ,KACN,SACA;CACE,uBAAuB,KAAK,IAAI,yBAAyB,CAAC,CAAC,KACzD,KAAK,WAAW,OAAO,IAAI,MAAM,OAAO,UAAU;EAAE,SAAS;EAAG,SAAS;CAAI,CAAC,CAAC,CAAC,GAChF,KAAK,gBACH,+EACF,GACA,KAAK,QACP;CACA,eAAe,KAAK,OAAO,gBAAgB,CAAC,CAAC,KAC3C,KAAK,YAAY,WAAW,GAC5B,KAAK,WAAW,OAAO,kBAAkB,GACzC,KAAK,QACF,aAAa,SAAS,WAAW,QAAQ,KAAK,SAAS,SAAS,QAAQ,SACnE,wCACR,GACA,KAAK,gBAAgB,yCAAyC,CAChE;CACA,SAAS,KAAK,IAAI,SAAS,CAAC,CAAC,KAC3B,KAAK,YAAY,EAAE,GACnB,KAAK,QACF,UAAU,SAAS,SACd,kCACR,GACA,KAAK,gBAAgB,+DAA+D,CACtF;AACF,IACC,EAAE,eAAe,SAAS,4BACzB,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,EACb,WAAW,OAAO,QAAQ,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,gBAAgB;EAAE;EAAM;CAAU,EAAE,EACrF;CACA,MAAM,SAAS,OAAOC,kBAAgC,MAAM;CAC5D,MAAM,MAAM,OAAOC;CACnB,MAAM,YAAY,OAAOC;CACzB,MAAM,UAAU,OAAO,eAAe,KAAK,EAAE,WAAW,IAAI,KAAK,CAAC;CAClE,MAAM,QAAQ;EAAE,UAAU;EAAe,OAAO;CAAQ;CACxD,MAAM,aAAa,OAAO,MAAM,uBAAuB;EACrD,cAAc,KAAA;EACd,SAAS,aAAa,EAAE,mCAAmC,OAAO,OAAO,EAAE;CAC7E,CAAC;CACD,MAAMC,YAAU,OAAO,oBAAoB,QAAQ,kBAAkB,SAAS,GAAG,KAAK;CAGtF,KAAK,MAAM,YAAY,OAAO,WAAW;EACvC,MAAM,UAAU,OAAO,SAAS;EAChC,IAAI,CAAC,SAAS;EACd,OAAO,YAAY,SAAS,WAAW;GACrC,SAAA;GACA,YAAY;GACZ,UAAU,QAAQ;GAClB;GACA;EACF,CAAC;EACD,OAAO,OAAO,QAAQ,YAAY,CAAC,GAAG,QAAQ,WAAW,QAAQ,IAAI,CAAC,CAAC,KAAK,GAAG,GAAG;CACpF;AACF,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,+DAA+D,CAAC"}

package/dist/{constants-D113Ut1x.js → constants-v3UX-hf6.mjs}

@@ -1,5 +1,4 @@
 import { HashSet } from "effect";
-
 //#region src/runtime-config/constants.ts
 /** Shared runtime keys owned by Airfoil rather than an individual connector. */
 const PlatformRuntimeKey = {
@@ -9,8 +8,9 @@ const PlatformRuntimeKey = {
 	connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
 	stateTable: "AIRFOIL_STATE_TABLE",
 	postgresConnectionString: "POSTGRES_CONNECTION_STRING",
-	wingsHost: "WINGS_HOST",
-	wingsNamespace: "WINGS_NAMESPACE",
+	engineUri: "ENGINE_URI",
+	flightUri: "FLIGHT_URI",
+	catalog: "AIRFOIL_CATALOG",
 	otelEnabled: "OTEL_ENABLED",
 	otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
 	otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS",
@@ -26,7 +26,7 @@ const PlatformRuntimeDefault = {
 const platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));
 /** @internal */
 const isPlatformRuntimeKey = (key) => HashSet.has(platformRuntimeKeys, key);
-
 //#endregion
 export { PlatformRuntimeKey as n, isPlatformRuntimeKey as r, PlatformRuntimeDefault as t };
-//# sourceMappingURL=constants-D113Ut1x.js.map
+
+//# sourceMappingURL=constants-v3UX-hf6.mjs.map

package/dist/constants-v3UX-hf6.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants-v3UX-hf6.mjs","names":[],"sources":["../src/runtime-config/constants.ts"],"sourcesContent":["import { HashSet } from \"effect\";\n\n/** Shared runtime keys owned by Airfoil rather than an individual connector. */\nexport const PlatformRuntimeKey = {\n  configPath: \"AIRFOIL_CONFIG_PATH\",\n  httpPort: \"AIRFOIL_HTTP_PORT\",\n  tableBindings: \"AIRFOIL_TABLE_BINDINGS\",\n  connectorInstanceId: \"AIRFOIL_CONNECTOR_INSTANCE_ID\",\n  stateTable: \"AIRFOIL_STATE_TABLE\",\n  postgresConnectionString: \"POSTGRES_CONNECTION_STRING\",\n  engineUri: \"ENGINE_URI\",\n  flightUri: \"FLIGHT_URI\",\n  catalog: \"AIRFOIL_CATALOG\",\n  otelEnabled: \"OTEL_ENABLED\",\n  otelExporterOtlpEndpoint: \"OTEL_EXPORTER_OTLP_ENDPOINT\",\n  otelExporterOtlpHeaders: \"OTEL_EXPORTER_OTLP_HEADERS\",\n  otelServiceName: \"OTEL_SERVICE_NAME\",\n  otelServiceVersion: \"OTEL_SERVICE_VERSION\",\n  otelResourceAttributes: \"OTEL_RESOURCE_ATTRIBUTES\",\n} as const;\n\n/** Defaults for optional platform-owned runtime configuration. */\nexport const PlatformRuntimeDefault = {\n  httpPort: 8080,\n  stateTable: \"_airfoil_connectors_state\",\n} as const;\n\nconst platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));\n\n/** @internal */\nexport const isPlatformRuntimeKey = (key: string): boolean => HashSet.has(platformRuntimeKeys, key);\n"],"mappings":";;;AAGA,MAAa,qBAAqB;CAChC,YAAY;CACZ,UAAU;CACV,eAAe;CACf,qBAAqB;CACrB,YAAY;CACZ,0BAA0B;CAC1B,WAAW;CACX,WAAW;CACX,SAAS;CACT,aAAa;CACb,0BAA0B;CAC1B,yBAAyB;CACzB,iBAAiB;CACjB,oBAAoB;CACpB,wBAAwB;AAC1B;;AAGA,MAAa,yBAAyB;CACpC,UAAU;CACV,YAAY;AACd;AAEA,MAAM,sBAAsB,QAAQ,aAAa,OAAO,OAAO,kBAAkB,CAAC;;AAGlF,MAAa,wBAAwB,QAAyB,QAAQ,IAAI,qBAAqB,GAAG"}

package/dist/demo/index.d.mts

@@ -0,0 +1,2 @@
+import { a as DemoDefinition, c as define, i as run, l as table, n as DemoTotals, o as DemoRows, r as RunOptions, s as DemoTable, u as DemoContext } from "../index-CV6V6fyx.mjs";
+export { type DemoContext, type DemoDefinition, type DemoRows, type DemoTable, type DemoTotals, type RunOptions, define, run, table };

package/dist/demo/index.mjs

@@ -0,0 +1,2 @@
+import { i as table, n as run, r as define } from "../demo-DqTJ7TWI.mjs";
+export { define, run, table };

package/dist/demo-DqTJ7TWI.mjs

@@ -0,0 +1,204 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { t as checkHasRepresentation } from "./table-CxM09xf7.mjs";
+import { t as Ingestor } from "./service-jWxxfEht.mjs";
+import { t as ingestBatch } from "./instrumented-Bq1WX2L5.mjs";
+import { Array as Array$1, Duration, Effect, Option, Schema, Stream } from "effect";
+import humanFormat from "human-format";
+import { Faker, base, en } from "@faker-js/faker";
+//#region src/demo/definition.ts
+const table = (definition) => definition;
+const define = (definition) => definition;
+//#endregion
+//#region src/demo/generate.ts
+const twoYears = Duration.toMillis(Duration.days(730));
+var Unsupported = class extends Error {};
+const unsupported = (path, what) => {
+	throw new Unsupported(`${path}: ${what} is not supported in demo data`);
+};
+/**
+* Compiles a row schema into a filler for demo rows. It reads the same
+* representation as `compileTable`, so values fit the Iceberg types. It fills
+* only required fields the row lacks, so optional ones such as `_deleted` stay
+* out.
+*/
+const compileRow = (schema) => Effect.try({
+	try: () => {
+		const { representation, references } = Schema.toRepresentation(Schema.toType(schema));
+		const resolve = (node, path) => {
+			if (node._tag !== "Reference") return node;
+			const target = references[node.$ref];
+			return target === void 0 ? unsupported(path, `reference ${node.$ref}`) : target;
+		};
+		const requiredFields = (node, path) => {
+			const resolved = resolve(node, path);
+			if (resolved._tag !== "Objects" || resolved.indexSignatures.length > 0) return unsupported(path, resolved._tag);
+			return resolved.propertySignatures.filter((property) => !property.isOptional).map((property) => {
+				const name = String(property.name);
+				return [name, value(property.type, `${path}.${name}`)];
+			});
+		};
+		const object = (node, path) => {
+			const fields = requiredFields(node, path);
+			return (context) => Object.fromEntries(fields.map(([name, make]) => [name, make(context)]));
+		};
+		const value = (node, path) => {
+			const resolved = resolve(node, path);
+			switch (resolved._tag) {
+				case "String": return ({ faker }) => faker.word.words({ count: {
+					min: 1,
+					max: 3
+				} });
+				case "Boolean": return ({ faker }) => faker.datatype.boolean();
+				case "BigInt": return ({ faker }) => BigInt(faker.number.int({ max: 1e5 }));
+				case "Number": return checkHasRepresentation(resolved.checks, "effect/schema/isInt") ? ({ faker }) => faker.number.int({ max: 1e3 }) : ({ faker }) => faker.number.float({
+					max: 1e3,
+					fractionDigits: 2
+				});
+				case "Literal": {
+					const literal = resolved.literal;
+					return () => literal;
+				}
+				case "Union": {
+					const members = resolved.types.filter((member) => member._tag !== "Null" && member._tag !== "Undefined");
+					const literals = members.flatMap((member) => member._tag === "Literal" ? [member.literal] : []);
+					if (literals.length > 0 && literals.length === members.length) return ({ faker }) => faker.helpers.arrayElement(literals);
+					const [only] = members;
+					return only !== void 0 && members.length === 1 ? value(only, path) : unsupported(path, "a union of several types");
+				}
+				case "Objects": return object(resolved, path);
+				case "Arrays": {
+					const [element] = resolved.rest;
+					if (resolved.elements.length > 0 || resolved.rest.length !== 1 || !element) return unsupported(path, "a tuple");
+					const item = value(element, `${path}[]`);
+					return (context) => Array.from({ length: context.faker.number.int({ max: 3 }) }, () => item(context));
+				}
+				case "Declaration": return declaration(resolved, path);
+				case "Suspend": return value(resolved.thunk, path);
+				default: return unsupported(path, resolved._tag);
+			}
+		};
+		const declaration = (node, path) => {
+			switch (node.representation?.id) {
+				case "effect/schema/Date": return ({ faker, now }) => faker.date.between({
+					from: now.getTime() - twoYears,
+					to: now
+				});
+				case "effect/schema/Uint8Array": return ({ faker }) => Uint8Array.from({ length: 8 }, () => faker.number.int(255));
+				case "effect/schema/ReadonlyMap": {
+					const [keyNode, valueNode] = node.typeParameters;
+					if (!keyNode || !valueNode) return unsupported(path, "a map without key or value");
+					const key = value(keyNode, `${path}.key`);
+					const item = value(valueNode, `${path}.value`);
+					return (context) => new Map(Array.from({ length: context.faker.number.int({ max: 3 }) }, () => [key(context), item(context)]));
+				}
+				default: return unsupported(path, node.representation?.id ?? "an unnamed declaration");
+			}
+		};
+		const fields = requiredFields(representation, "$");
+		return (context, row) => ({
+			...Object.fromEntries(fields.filter(([name]) => !(name in row)).map(([name, make]) => [name, make(context)])),
+			...row
+		});
+	},
+	catch: (cause) => new ConnectorError({
+		message: cause instanceof Unsupported ? cause.message : "Failed to compile demo rows",
+		cause
+	})
+});
+//#endregion
+//#region src/demo/run.ts
+const empty = {
+	rows: 0,
+	bytes: 0,
+	tables: {}
+};
+const sum = (results) => ({
+	rows: results.reduce((total, result) => total + result.rows, 0),
+	bytes: results.reduce((total, result) => total + result.bytes, 0)
+});
+const add = (totals, written) => written.reduce((current, [name, result]) => ({
+	rows: current.rows + result.rows,
+	bytes: current.bytes + result.bytes,
+	tables: {
+		...current.tables,
+		[name]: {
+			rows: (current.tables[name]?.rows ?? 0) + result.rows,
+			bytes: (current.tables[name]?.bytes ?? 0) + result.bytes
+		}
+	}
+}), totals);
+/**
+* Writes the setup rows, then groups, until it reaches the size or the demo's
+* count. It can go over the size by one chunk. Only bound tables are
+* written; keys still point to the others.
+*/
+const run = Effect.fnUntraced(function* (demo, options) {
+	const ingestor = yield* Ingestor;
+	const batchSize = yield* Schema.decodeUnknownEffect(Schema.Int.check(Schema.isGreaterThan(0)))(options.batchSize ?? 500).pipe(Effect.mapError((cause) => new ConnectorError({
+		message: "Invalid demo batch size",
+		cause
+	})));
+	const bound = (table) => ingestor.resourceNames === void 0 || ingestor.resourceNames.includes(table.name);
+	const faker = new Faker({ locale: [en, base] });
+	faker.seed(options.seed);
+	const context = {
+		faker,
+		now: options.now
+	};
+	const prepare = (tables) => Effect.forEach(tables.filter(bound), (table) => compileRow(table.rowSchema).pipe(Effect.map((fill) => ({
+		table,
+		fill,
+		decode: Schema.decodeUnknownEffect(Schema.toType(table.rowSchema))
+	}))));
+	const write = (tables, rowsOf) => Effect.gen(function* () {
+		const prepared = yield* Effect.forEach(tables, ({ table, fill, decode }) => Effect.forEach(rowsOf(table.name), (row) => decode(fill(context, row)).pipe(Effect.mapError((cause) => new ConnectorError({
+			message: `Invalid demo row for ${table.name}`,
+			cause
+		})))).pipe(Effect.map((rows) => ({
+			table,
+			rows
+		}))));
+		return yield* Effect.forEach(prepared, ({ table, rows }) => Effect.forEach(Array$1.chunksOf(rows, batchSize), (rows) => ingestBatch({
+			connector: demo.name,
+			resource: table.name,
+			source: "demo",
+			batch: { rows }
+		}), { concurrency: 8 }).pipe(Effect.map((results) => [table.name, sum(results)])), { concurrency: 4 });
+	});
+	const setupTables = yield* prepare(demo.setup.tables);
+	const groupTables = yield* prepare(demo.group.tables);
+	const setup = demo.setup.rows(context);
+	const afterSetup = add(empty, yield* write(setupTables, (name) => setup[name] ?? []));
+	const totals = afterSetup.bytes >= options.bytes || groupTables.length === 0 ? afterSetup : yield* Stream.range(0, (demo.group.count ?? Infinity) - 1).pipe(Stream.grouped(options.groupsPerChunk ?? demo.group.groupsPerChunk ?? 1e3), Stream.mapEffect((indexes) => {
+		const groups = indexes.map((index) => demo.group.rows(index, context));
+		return write(groupTables, (name) => groups.flatMap((group) => group[name] ?? []));
+	}), Stream.scan(() => afterSetup, add), Stream.takeUntil((current) => current.bytes >= options.bytes), Stream.tap((current) => Effect.logInfo("demo progress").pipe(Effect.annotateLogs({
+		rows: current.rows,
+		size: humanFormat.bytes(current.bytes)
+	}))), Stream.runLast, Effect.map(Option.getOrElse(() => afterSetup)));
+	if (totals.bytes < options.bytes) yield* Effect.logWarning("The selected tables ran out of rows, so the run wrote less than the size asked for").pipe(Effect.annotateLogs({
+		size: humanFormat.bytes(totals.bytes),
+		target: humanFormat.bytes(options.bytes)
+	}));
+	yield* Effect.logInfo("demo done").pipe(Effect.annotateLogs({
+		rows: totals.rows,
+		size: humanFormat.bytes(totals.bytes),
+		tables: Object.fromEntries(Object.entries(totals.tables).map(([name, result]) => [name, {
+			rows: result.rows,
+			size: humanFormat.bytes(result.bytes)
+		}]))
+	}));
+	return totals;
+});
+//#endregion
+//#region src/demo/index.ts
+var demo_exports = /* @__PURE__ */ __exportAll({
+	define: () => define,
+	run: () => run,
+	table: () => table
+});
+//#endregion
+export { table as i, run as n, define as r, demo_exports as t };
+
+//# sourceMappingURL=demo-DqTJ7TWI.mjs.map

package/dist/demo-DqTJ7TWI.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"demo-DqTJ7TWI.mjs","names":["Arr"],"sources":["../src/demo/definition.ts","../src/demo/generate.ts","../src/demo/run.ts","../src/demo/index.ts"],"sourcesContent":["import type { Schema } from \"effect\";\n\nimport type { ResourceRequiredField, ResourceSchema } from \"../core/types\";\nimport type { DemoContext } from \"./generate\";\n\n/** A table a demo writes: the same name, row schema and version as its resource. */\nexport type DemoTable<\n  Name extends string = string,\n  S extends ResourceSchema = ResourceSchema,\n  Version extends string = string,\n> = {\n  readonly name: Name;\n  readonly rowSchema: S;\n  readonly version: Version;\n};\n\nexport const table = <\n  const Name extends string,\n  S extends ResourceSchema,\n  const Version extends ResourceRequiredField<S>,\n>(\n  definition: DemoTable<Name, S, Version>,\n): DemoTable<Name, S, Version> => definition;\n\n/** Rows by table. Required fields a row leaves out get placeholder values. */\nexport type DemoRows<Tables extends ReadonlyArray<DemoTable>> = {\n  readonly [Table in Tables[number] as Table[\"name\"]]?: ReadonlyArray<\n    Partial<Schema.Schema.Type<Table[\"rowSchema\"]>>\n  >;\n};\n\nexport type DemoDefinition<\n  Setup extends ReadonlyArray<DemoTable> = ReadonlyArray<DemoTable>,\n  Group extends ReadonlyArray<DemoTable> = ReadonlyArray<DemoTable>,\n> = {\n  readonly name: string;\n  /** Tables written once, before any group, such as a product catalog. */\n  readonly setup: {\n    readonly tables: Setup;\n    readonly rows: (context: DemoContext) => DemoRows<Setup>;\n  };\n  /**\n   * Tables for one related unit, such as a customer and their orders. The run\n   * adds groups until it reaches the size. Build keys from the index, so they\n   * are unique and other groups can point to them.\n   */\n  readonly group: {\n    readonly tables: Group;\n    readonly rows: (index: number, context: DemoContext) => DemoRows<Group>;\n    /**\n     * Groups written per chunk, 1000 by default. A run can go over the size by\n     * one chunk, so lower it when a group is big.\n     */\n    readonly groupsPerChunk?: number;\n    /**\n     * The number of groups there are, when the provider only keeps so much\n     * history. The run stops there, even below the size.\n     */\n    readonly count?: number;\n  };\n};\n\nexport const define = <\n  const Setup extends ReadonlyArray<DemoTable>,\n  const Group extends ReadonlyArray<DemoTable>,\n>(\n  definition: DemoDefinition<Setup, Group>,\n): DemoDefinition<Setup, Group> => definition;\n","import type { Faker } from \"@faker-js/faker\";\n\nimport { Duration, Effect, Schema, type SchemaRepresentation } from \"effect\";\n\nimport { ConnectorError } from \"../errors\";\nimport { checkHasRepresentation } from \"../iceberg/table\";\n\n/** What one run shares: the Faker and the time the run started. */\nexport type DemoContext = {\n  readonly faker: Faker;\n  readonly now: Date;\n};\n\n/** Completes a row: keeps what it has, and fills the required fields it lacks. */\nexport type RowFiller = (context: DemoContext, row: object) => Record<string, unknown>;\n\ntype Representation = SchemaRepresentation.Representation;\ntype Value = (context: DemoContext) => unknown;\n\nconst twoYears = Duration.toMillis(Duration.days(730));\n\nclass Unsupported extends Error {}\n\nconst unsupported = (path: string, what: string): never => {\n  throw new Unsupported(`${path}: ${what} is not supported in demo data`);\n};\n\n/**\n * Compiles a row schema into a filler for demo rows. It reads the same\n * representation as `compileTable`, so values fit the Iceberg types. It fills\n * only required fields the row lacks, so optional ones such as `_deleted` stay\n * out.\n */\nexport const compileRow = (schema: Schema.Top): Effect.Effect<RowFiller, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      const { representation, references } = Schema.toRepresentation(Schema.toType(schema));\n\n      const resolve = (node: Representation, path: string): Representation => {\n        if (node._tag !== \"Reference\") return node;\n        const target = references[node.$ref];\n\n        return target === undefined ? unsupported(path, `reference ${node.$ref}`) : target;\n      };\n\n      const requiredFields = (node: Representation, path: string) => {\n        const resolved = resolve(node, path);\n\n        if (resolved._tag !== \"Objects\" || resolved.indexSignatures.length > 0) {\n          return unsupported(path, resolved._tag);\n        }\n\n        return resolved.propertySignatures\n          .filter((property) => !property.isOptional)\n          .map((property) => {\n            const name = String(property.name);\n            return [name, value(property.type, `${path}.${name}`)] as const;\n          });\n      };\n\n      const object = (node: Representation, path: string): Value => {\n        const fields = requiredFields(node, path);\n        return (context) => Object.fromEntries(fields.map(([name, make]) => [name, make(context)]));\n      };\n\n      const value = (node: Representation, path: string): Value => {\n        const resolved = resolve(node, path);\n\n        switch (resolved._tag) {\n          case \"String\":\n            return ({ faker }) => faker.word.words({ count: { min: 1, max: 3 } });\n          case \"Boolean\":\n            return ({ faker }) => faker.datatype.boolean();\n          case \"BigInt\":\n            return ({ faker }) => BigInt(faker.number.int({ max: 100_000 }));\n          case \"Number\":\n            return checkHasRepresentation(resolved.checks, \"effect/schema/isInt\")\n              ? ({ faker }) => faker.number.int({ max: 1000 })\n              : ({ faker }) => faker.number.float({ max: 1000, fractionDigits: 2 });\n          case \"Literal\": {\n            const literal = resolved.literal;\n            return () => literal;\n          }\n          case \"Union\": {\n            // Fill every field, so nullable unions get their value type.\n            const members = resolved.types.filter(\n              (member) => member._tag !== \"Null\" && member._tag !== \"Undefined\",\n            );\n\n            const literals = members.flatMap((member) =>\n              member._tag === \"Literal\" ? [member.literal] : [],\n            );\n\n            if (literals.length > 0 && literals.length === members.length) {\n              return ({ faker }) => faker.helpers.arrayElement(literals);\n            }\n\n            const [only] = members;\n\n            return only !== undefined && members.length === 1\n              ? value(only, path)\n              : unsupported(path, \"a union of several types\");\n          }\n          case \"Objects\":\n            return object(resolved, path);\n          case \"Arrays\": {\n            const [element] = resolved.rest;\n\n            if (resolved.elements.length > 0 || resolved.rest.length !== 1 || !element) {\n              return unsupported(path, \"a tuple\");\n            }\n\n            const item = value(element, `${path}[]`);\n\n            return (context) =>\n              Array.from({ length: context.faker.number.int({ max: 3 }) }, () => item(context));\n          }\n          case \"Declaration\":\n            return declaration(resolved, path);\n          case \"Suspend\":\n            return value(resolved.thunk, path);\n          default:\n            return unsupported(path, resolved._tag);\n        }\n      };\n\n      const declaration = (node: SchemaRepresentation.Declaration, path: string): Value => {\n        switch (node.representation?.id) {\n          case \"effect/schema/Date\":\n            return ({ faker, now }) =>\n              faker.date.between({ from: now.getTime() - twoYears, to: now });\n          case \"effect/schema/Uint8Array\":\n            return ({ faker }) => Uint8Array.from({ length: 8 }, () => faker.number.int(255));\n          case \"effect/schema/ReadonlyMap\": {\n            const [keyNode, valueNode] = node.typeParameters;\n            if (!keyNode || !valueNode) return unsupported(path, \"a map without key or value\");\n            const key = value(keyNode, `${path}.key`);\n            const item = value(valueNode, `${path}.value`);\n\n            return (context) =>\n              new Map(\n                Array.from({ length: context.faker.number.int({ max: 3 }) }, () => [\n                  key(context),\n                  item(context),\n                ]),\n              );\n          }\n          default:\n            return unsupported(path, node.representation?.id ?? \"an unnamed declaration\");\n        }\n      };\n\n      const fields = requiredFields(representation, \"$\");\n\n      return (context, row) => ({\n        ...Object.fromEntries(\n          fields.filter(([name]) => !(name in row)).map(([name, make]) => [name, make(context)]),\n        ),\n        ...row,\n      });\n    },\n    catch: (cause) =>\n      new ConnectorError({\n        message: cause instanceof Unsupported ? cause.message : \"Failed to compile demo rows\",\n        cause,\n      }),\n  });\n","import { Faker, base, en } from \"@faker-js/faker\";\nimport { Array as Arr, Effect, Option, Schema, Stream } from \"effect\";\nimport humanFormat from \"human-format\";\n\nimport type { DemoDefinition, DemoTable } from \"./definition\";\n\nimport { ConnectorError } from \"../errors\";\nimport { ingestBatch } from \"../ingestor/instrumented\";\nimport { Ingestor, type IngestResult } from \"../ingestor/service\";\nimport { compileRow, type DemoContext } from \"./generate\";\n\nexport type RunOptions = {\n  /** Stop once this many Arrow bytes are written. */\n  readonly bytes: number;\n  /** Omit for random data on each run. */\n  readonly seed?: number;\n  /** The time data is made around. Same seed, time and tables give the same rows. */\n  readonly now: Date;\n  /** Maximum rows per ingestion batch. Defaults to 500. Must be a positive integer. */\n  readonly batchSize?: number;\n  /** Groups per chunk. Overrides the demo's. */\n  readonly groupsPerChunk?: number;\n};\n\nexport type DemoTotals = IngestResult & {\n  readonly tables: Readonly<Record<string, IngestResult>>;\n};\n\nconst empty: DemoTotals = { rows: 0, bytes: 0, tables: {} };\n\nconst sum = (results: ReadonlyArray<IngestResult>): IngestResult => ({\n  rows: results.reduce((total, result) => total + result.rows, 0),\n  bytes: results.reduce((total, result) => total + result.bytes, 0),\n});\n\nconst add = (totals: DemoTotals, written: ReadonlyArray<readonly [string, IngestResult]>) =>\n  written.reduce<DemoTotals>(\n    (current, [name, result]) => ({\n      rows: current.rows + result.rows,\n      bytes: current.bytes + result.bytes,\n      tables: {\n        ...current.tables,\n        [name]: {\n          rows: (current.tables[name]?.rows ?? 0) + result.rows,\n          bytes: (current.tables[name]?.bytes ?? 0) + result.bytes,\n        },\n      },\n    }),\n    totals,\n  );\n\n/**\n * Writes the setup rows, then groups, until it reaches the size or the demo's\n * count. It can go over the size by one chunk. Only bound tables are\n * written; keys still point to the others.\n */\nexport const run = Effect.fnUntraced(function* (demo: DemoDefinition, options: RunOptions) {\n  const ingestor = yield* Ingestor;\n  // Wide rows may need a lower batch size to fit the engine's limits.\n  const batchSize = yield* Schema.decodeUnknownEffect(Schema.Int.check(Schema.isGreaterThan(0)))(\n    options.batchSize ?? 500,\n  ).pipe(\n    Effect.mapError((cause) => new ConnectorError({ message: \"Invalid demo batch size\", cause })),\n  );\n\n  const bound = (table: DemoTable) =>\n    ingestor.resourceNames === undefined || ingestor.resourceNames.includes(table.name);\n\n  // A fresh Faker per run keeps explicit seeds repeatable.\n  const faker = new Faker({ locale: [en, base] });\n  faker.seed(options.seed);\n  const context: DemoContext = { faker, now: options.now };\n\n  const prepare = (tables: ReadonlyArray<DemoTable>) =>\n    Effect.forEach(tables.filter(bound), (table) =>\n      compileRow(table.rowSchema).pipe(\n        Effect.map((fill) => ({\n          table,\n          fill,\n          decode: Schema.decodeUnknownEffect(Schema.toType(table.rowSchema)),\n        })),\n      ),\n    );\n\n  const write = (\n    tables: Effect.Success<ReturnType<typeof prepare>>,\n    rowsOf: (name: string) => ReadonlyArray<object>,\n  ) =>\n    Effect.gen(function* () {\n      // Fill rows in table order before concurrent writes, keeping Faker deterministic.\n      const prepared = yield* Effect.forEach(tables, ({ table, fill, decode }) =>\n        Effect.forEach(rowsOf(table.name), (row) =>\n          decode(fill(context, row)).pipe(\n            Effect.mapError(\n              (cause) =>\n                new ConnectorError({ message: `Invalid demo row for ${table.name}`, cause }),\n            ),\n          ),\n        ).pipe(Effect.map((rows) => ({ table, rows }))),\n      );\n\n      // Share durable flushes with bounded parallel writes.\n      return yield* Effect.forEach(\n        prepared,\n        ({ table, rows }) =>\n          Effect.forEach(\n            Arr.chunksOf(rows, batchSize),\n            (rows) =>\n              ingestBatch({\n                connector: demo.name,\n                resource: table.name,\n                source: \"demo\",\n                batch: { rows },\n              }),\n            { concurrency: 8 },\n          ).pipe(Effect.map((results) => [table.name, sum(results)] as const)),\n        { concurrency: 4 },\n      );\n    });\n\n  const setupTables = yield* prepare(demo.setup.tables);\n  const groupTables = yield* prepare(demo.group.tables);\n\n  const setup = demo.setup.rows(context);\n  const afterSetup = add(empty, yield* write(setupTables, (name) => setup[name] ?? []));\n\n  const totals =\n    afterSetup.bytes >= options.bytes || groupTables.length === 0\n      ? afterSetup\n      : yield* Stream.range(0, (demo.group.count ?? Infinity) - 1).pipe(\n          Stream.grouped(options.groupsPerChunk ?? demo.group.groupsPerChunk ?? 1000),\n          Stream.mapEffect((indexes) => {\n            const groups = indexes.map((index) => demo.group.rows(index, context));\n            return write(groupTables, (name) => groups.flatMap((group) => group[name] ?? []));\n          }),\n          Stream.scan(() => afterSetup, add),\n          Stream.takeUntil((current) => current.bytes >= options.bytes),\n          Stream.tap((current) =>\n            Effect.logInfo(\"demo progress\").pipe(\n              Effect.annotateLogs({ rows: current.rows, size: humanFormat.bytes(current.bytes) }),\n            ),\n          ),\n          Stream.runLast,\n          Effect.map(Option.getOrElse(() => afterSetup)),\n        );\n\n  if (totals.bytes < options.bytes) {\n    yield* Effect.logWarning(\n      \"The selected tables ran out of rows, so the run wrote less than the size asked for\",\n    ).pipe(\n      Effect.annotateLogs({\n        size: humanFormat.bytes(totals.bytes),\n        target: humanFormat.bytes(options.bytes),\n      }),\n    );\n  }\n\n  yield* Effect.logInfo(\"demo done\").pipe(\n    Effect.annotateLogs({\n      rows: totals.rows,\n      size: humanFormat.bytes(totals.bytes),\n      tables: Object.fromEntries(\n        Object.entries(totals.tables).map(([name, result]) => [\n          name,\n          { rows: result.rows, size: humanFormat.bytes(result.bytes) },\n        ]),\n      ),\n    }),\n  );\n\n  return totals;\n});\n","export { type DemoContext } from \"./generate\";\nexport { define, table, type DemoDefinition, type DemoRows, type DemoTable } from \"./definition\";\nexport { run, type DemoTotals, type RunOptions } from \"./run\";\n"],"mappings":";;;;;;;;;AAgBA,MAAa,SAKX,eACgC;AAwClC,MAAa,UAIX,eACiC;;;AChDnC,MAAM,WAAW,SAAS,SAAS,SAAS,KAAK,GAAG,CAAC;AAErD,IAAM,cAAN,cAA0B,MAAM,CAAC;AAEjC,MAAM,eAAe,MAAc,SAAwB;CACzD,MAAM,IAAI,YAAY,GAAG,KAAK,IAAI,KAAK,+BAA+B;AACxE;;;;;;;AAQA,MAAa,cAAc,WACzB,OAAO,IAAI;CACT,WAAW;EACT,MAAM,EAAE,gBAAgB,eAAe,OAAO,iBAAiB,OAAO,OAAO,MAAM,CAAC;EAEpF,MAAM,WAAW,MAAsB,SAAiC;GACtE,IAAI,KAAK,SAAS,aAAa,OAAO;GACtC,MAAM,SAAS,WAAW,KAAK;GAE/B,OAAO,WAAW,KAAA,IAAY,YAAY,MAAM,aAAa,KAAK,MAAM,IAAI;EAC9E;EAEA,MAAM,kBAAkB,MAAsB,SAAiB;GAC7D,MAAM,WAAW,QAAQ,MAAM,IAAI;GAEnC,IAAI,SAAS,SAAS,aAAa,SAAS,gBAAgB,SAAS,GACnE,OAAO,YAAY,MAAM,SAAS,IAAI;GAGxC,OAAO,SAAS,mBACb,QAAQ,aAAa,CAAC,SAAS,UAAU,CAAC,CAC1C,KAAK,aAAa;IACjB,MAAM,OAAO,OAAO,SAAS,IAAI;IACjC,OAAO,CAAC,MAAM,MAAM,SAAS,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;GACvD,CAAC;EACL;EAEA,MAAM,UAAU,MAAsB,SAAwB;GAC5D,MAAM,SAAS,eAAe,MAAM,IAAI;GACxC,QAAQ,YAAY,OAAO,YAAY,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC;EAC5F;EAEA,MAAM,SAAS,MAAsB,SAAwB;GAC3D,MAAM,WAAW,QAAQ,MAAM,IAAI;GAEnC,QAAQ,SAAS,MAAjB;IACE,KAAK,UACH,QAAQ,EAAE,YAAY,MAAM,KAAK,MAAM,EAAE,OAAO;KAAE,KAAK;KAAG,KAAK;IAAE,EAAE,CAAC;IACtE,KAAK,WACH,QAAQ,EAAE,YAAY,MAAM,SAAS,QAAQ;IAC/C,KAAK,UACH,QAAQ,EAAE,YAAY,OAAO,MAAM,OAAO,IAAI,EAAE,KAAK,IAAQ,CAAC,CAAC;IACjE,KAAK,UACH,OAAO,uBAAuB,SAAS,QAAQ,qBAAqB,KAC/D,EAAE,YAAY,MAAM,OAAO,IAAI,EAAE,KAAK,IAAK,CAAC,KAC5C,EAAE,YAAY,MAAM,OAAO,MAAM;KAAE,KAAK;KAAM,gBAAgB;IAAE,CAAC;IACxE,KAAK,WAAW;KACd,MAAM,UAAU,SAAS;KACzB,aAAa;IACf;IACA,KAAK,SAAS;KAEZ,MAAM,UAAU,SAAS,MAAM,QAC5B,WAAW,OAAO,SAAS,UAAU,OAAO,SAAS,WACxD;KAEA,MAAM,WAAW,QAAQ,SAAS,WAChC,OAAO,SAAS,YAAY,CAAC,OAAO,OAAO,IAAI,CAAC,CAClD;KAEA,IAAI,SAAS,SAAS,KAAK,SAAS,WAAW,QAAQ,QACrD,QAAQ,EAAE,YAAY,MAAM,QAAQ,aAAa,QAAQ;KAG3D,MAAM,CAAC,QAAQ;KAEf,OAAO,SAAS,KAAA,KAAa,QAAQ,WAAW,IAC5C,MAAM,MAAM,IAAI,IAChB,YAAY,MAAM,0BAA0B;IAClD;IACA,KAAK,WACH,OAAO,OAAO,UAAU,IAAI;IAC9B,KAAK,UAAU;KACb,MAAM,CAAC,WAAW,SAAS;KAE3B,IAAI,SAAS,SAAS,SAAS,KAAK,SAAS,KAAK,WAAW,KAAK,CAAC,SACjE,OAAO,YAAY,MAAM,SAAS;KAGpC,MAAM,OAAO,MAAM,SAAS,GAAG,KAAK,GAAG;KAEvC,QAAQ,YACN,MAAM,KAAK,EAAE,QAAQ,QAAQ,MAAM,OAAO,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,KAAK,OAAO,CAAC;IACpF;IACA,KAAK,eACH,OAAO,YAAY,UAAU,IAAI;IACnC,KAAK,WACH,OAAO,MAAM,SAAS,OAAO,IAAI;IACnC,SACE,OAAO,YAAY,MAAM,SAAS,IAAI;GAC1C;EACF;EAEA,MAAM,eAAe,MAAwC,SAAwB;GACnF,QAAQ,KAAK,gBAAgB,IAA7B;IACE,KAAK,sBACH,QAAQ,EAAE,OAAO,UACf,MAAM,KAAK,QAAQ;KAAE,MAAM,IAAI,QAAQ,IAAI;KAAU,IAAI;IAAI,CAAC;IAClE,KAAK,4BACH,QAAQ,EAAE,YAAY,WAAW,KAAK,EAAE,QAAQ,EAAE,SAAS,MAAM,OAAO,IAAI,GAAG,CAAC;IAClF,KAAK,6BAA6B;KAChC,MAAM,CAAC,SAAS,aAAa,KAAK;KAClC,IAAI,CAAC,WAAW,CAAC,WAAW,OAAO,YAAY,MAAM,4BAA4B;KACjF,MAAM,MAAM,MAAM,SAAS,GAAG,KAAK,KAAK;KACxC,MAAM,OAAO,MAAM,WAAW,GAAG,KAAK,OAAO;KAE7C,QAAQ,YACN,IAAI,IACF,MAAM,KAAK,EAAE,QAAQ,QAAQ,MAAM,OAAO,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,CACjE,IAAI,OAAO,GACX,KAAK,OAAO,CACd,CAAC,CACH;IACJ;IACA,SACE,OAAO,YAAY,MAAM,KAAK,gBAAgB,MAAM,wBAAwB;GAChF;EACF;EAEA,MAAM,SAAS,eAAe,gBAAgB,GAAG;EAEjD,QAAQ,SAAS,SAAS;GACxB,GAAG,OAAO,YACR,OAAO,QAAQ,CAAC,UAAU,EAAE,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,UAAU,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CACvF;GACA,GAAG;EACL;CACF;CACA,QAAQ,UACN,IAAI,eAAe;EACjB,SAAS,iBAAiB,cAAc,MAAM,UAAU;EACxD;CACF,CAAC;AACL,CAAC;;;AC1IH,MAAM,QAAoB;CAAE,MAAM;CAAG,OAAO;CAAG,QAAQ,CAAC;AAAE;AAE1D,MAAM,OAAO,aAAwD;CACnE,MAAM,QAAQ,QAAQ,OAAO,WAAW,QAAQ,OAAO,MAAM,CAAC;CAC9D,OAAO,QAAQ,QAAQ,OAAO,WAAW,QAAQ,OAAO,OAAO,CAAC;AAClE;AAEA,MAAM,OAAO,QAAoB,YAC/B,QAAQ,QACL,SAAS,CAAC,MAAM,aAAa;CAC5B,MAAM,QAAQ,OAAO,OAAO;CAC5B,OAAO,QAAQ,QAAQ,OAAO;CAC9B,QAAQ;EACN,GAAG,QAAQ;GACV,OAAO;GACN,OAAO,QAAQ,OAAO,KAAK,EAAE,QAAQ,KAAK,OAAO;GACjD,QAAQ,QAAQ,OAAO,KAAK,EAAE,SAAS,KAAK,OAAO;EACrD;CACF;AACF,IACA,MACF;;;;;;AAOF,MAAa,MAAM,OAAO,WAAW,WAAW,MAAsB,SAAqB;CACzF,MAAM,WAAW,OAAO;CAExB,MAAM,YAAY,OAAO,OAAO,oBAAoB,OAAO,IAAI,MAAM,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,CAC5F,QAAQ,aAAa,GACvB,CAAC,CAAC,KACA,OAAO,UAAU,UAAU,IAAI,eAAe;EAAE,SAAS;EAA2B;CAAM,CAAC,CAAC,CAC9F;CAEA,MAAM,SAAS,UACb,SAAS,kBAAkB,KAAA,KAAa,SAAS,cAAc,SAAS,MAAM,IAAI;CAGpF,MAAM,QAAQ,IAAI,MAAM,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;CAC9C,MAAM,KAAK,QAAQ,IAAI;CACvB,MAAM,UAAuB;EAAE;EAAO,KAAK,QAAQ;CAAI;CAEvD,MAAM,WAAW,WACf,OAAO,QAAQ,OAAO,OAAO,KAAK,IAAI,UACpC,WAAW,MAAM,SAAS,CAAC,CAAC,KAC1B,OAAO,KAAK,UAAU;EACpB;EACA;EACA,QAAQ,OAAO,oBAAoB,OAAO,OAAO,MAAM,SAAS,CAAC;CACnE,EAAE,CACJ,CACF;CAEF,MAAM,SACJ,QACA,WAEA,OAAO,IAAI,aAAa;EAEtB,MAAM,WAAW,OAAO,OAAO,QAAQ,SAAS,EAAE,OAAO,MAAM,aAC7D,OAAO,QAAQ,OAAO,MAAM,IAAI,IAAI,QAClC,OAAO,KAAK,SAAS,GAAG,CAAC,CAAC,CAAC,KACzB,OAAO,UACJ,UACC,IAAI,eAAe;GAAE,SAAS,wBAAwB,MAAM;GAAQ;EAAM,CAAC,CAC/E,CACF,CACF,CAAC,CAAC,KAAK,OAAO,KAAK,UAAU;GAAE;GAAO;EAAK,EAAE,CAAC,CAChD;EAGA,OAAO,OAAO,OAAO,QACnB,WACC,EAAE,OAAO,WACR,OAAO,QACLA,QAAI,SAAS,MAAM,SAAS,IAC3B,SACC,YAAY;GACV,WAAW,KAAK;GAChB,UAAU,MAAM;GAChB,QAAQ;GACR,OAAO,EAAE,KAAK;EAChB,CAAC,GACH,EAAE,aAAa,EAAE,CACnB,CAAC,CAAC,KAAK,OAAO,KAAK,YAAY,CAAC,MAAM,MAAM,IAAI,OAAO,CAAC,CAAU,CAAC,GACrE,EAAE,aAAa,EAAE,CACnB;CACF,CAAC;CAEH,MAAM,cAAc,OAAO,QAAQ,KAAK,MAAM,MAAM;CACpD,MAAM,cAAc,OAAO,QAAQ,KAAK,MAAM,MAAM;CAEpD,MAAM,QAAQ,KAAK,MAAM,KAAK,OAAO;CACrC,MAAM,aAAa,IAAI,OAAO,OAAO,MAAM,cAAc,SAAS,MAAM,SAAS,CAAC,CAAC,CAAC;CAEpF,MAAM,SACJ,WAAW,SAAS,QAAQ,SAAS,YAAY,WAAW,IACxD,aACA,OAAO,OAAO,MAAM,IAAI,KAAK,MAAM,SAAS,YAAY,CAAC,CAAC,CAAC,KACzD,OAAO,QAAQ,QAAQ,kBAAkB,KAAK,MAAM,kBAAkB,GAAI,GAC1E,OAAO,WAAW,YAAY;EAC5B,MAAM,SAAS,QAAQ,KAAK,UAAU,KAAK,MAAM,KAAK,OAAO,OAAO,CAAC;EACrE,OAAO,MAAM,cAAc,SAAS,OAAO,SAAS,UAAU,MAAM,SAAS,CAAC,CAAC,CAAC;CAClF,CAAC,GACD,OAAO,WAAW,YAAY,GAAG,GACjC,OAAO,WAAW,YAAY,QAAQ,SAAS,QAAQ,KAAK,GAC5D,OAAO,KAAK,YACV,OAAO,QAAQ,eAAe,CAAC,CAAC,KAC9B,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM,MAAM,YAAY,MAAM,QAAQ,KAAK;CAAE,CAAC,CACpF,CACF,GACA,OAAO,SACP,OAAO,IAAI,OAAO,gBAAgB,UAAU,CAAC,CAC/C;CAEN,IAAI,OAAO,QAAQ,QAAQ,OACzB,OAAO,OAAO,WACZ,oFACF,CAAC,CAAC,KACA,OAAO,aAAa;EAClB,MAAM,YAAY,MAAM,OAAO,KAAK;EACpC,QAAQ,YAAY,MAAM,QAAQ,KAAK;CACzC,CAAC,CACH;CAGF,OAAO,OAAO,QAAQ,WAAW,CAAC,CAAC,KACjC,OAAO,aAAa;EAClB,MAAM,OAAO;EACb,MAAM,YAAY,MAAM,OAAO,KAAK;EACpC,QAAQ,OAAO,YACb,OAAO,QAAQ,OAAO,MAAM,CAAC,CAAC,KAAK,CAAC,MAAM,YAAY,CACpD,MACA;GAAE,MAAM,OAAO;GAAM,MAAM,YAAY,MAAM,OAAO,KAAK;EAAE,CAC7D,CAAC,CACH;CACF,CAAC,CACH;CAEA,OAAO;AACT,CAAC"}

package/dist/errors/index.d.mts

@@ -0,0 +1,2 @@
+import { t as ConnectorError } from "../index-DkVNO6ks.mjs";
+export { ConnectorError };

package/dist/errors/index.mjs

@@ -0,0 +1,2 @@
+import { t as ConnectorError } from "../errors-pT0YVJU5.mjs";
+export { ConnectorError };

package/dist/{errors-DHWylqRZ.js → errors-pT0YVJU5.mjs}

@@ -1,8 +1,7 @@
 import { Data } from "effect";
-
 //#region src/errors/connector-error.ts
 var ConnectorError = class extends Data.TaggedError("ConnectorError") {};
-
 //#endregion
 export { ConnectorError as t };
-//# sourceMappingURL=errors-DHWylqRZ.js.map
+
+//# sourceMappingURL=errors-pT0YVJU5.mjs.map

package/dist/errors-pT0YVJU5.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors-pT0YVJU5.mjs","names":[],"sources":["../src/errors/connector-error.ts"],"sourcesContent":["import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n  readonly message: string;\n  readonly cause?: unknown;\n}> {}\n"],"mappings":";;AAEA,IAAa,iBAAb,cAAoC,KAAK,YAAY,gBAAgB,CAAC,CAGnE,CAAC"}

package/dist/iceberg/index.d.mts

@@ -0,0 +1,3 @@
+import { r as identifier } from "../annotations-vzSzXtr-.mjs";
+import { i as IcebergSchemaError, n as CreateTableOptions, r as makeCreateTableRequest } from "../index-BTAbexBp.mjs";
+export { type CreateTableOptions, IcebergSchemaError, identifier, makeCreateTableRequest };

package/dist/iceberg/index.mjs

@@ -0,0 +1,3 @@
+import { a as IcebergSchemaError, i as makeCreateTableRequest } from "../table-CxM09xf7.mjs";
+import { n as identifier } from "../iceberg-Cj2PwlNm.mjs";
+export { IcebergSchemaError, identifier, makeCreateTableRequest };

package/dist/iceberg-Cj2PwlNm.mjs

@@ -0,0 +1,30 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { a as IcebergSchemaError, i as makeCreateTableRequest } from "./table-CxM09xf7.mjs";
+import "effect";
+//#region src/iceberg/annotations.ts
+/**
+* Marks the fields that identify a row. The engine upserts rows by them.
+*
+* Put it last. A later `.annotate()` keeps the runtime annotation, but the type
+* loses the key, so `Resource.entity` stops compiling.
+*
+* @example
+* ```ts
+* const Post = Schema.Struct({
+*   id: Schema.String,
+*   updatedAt: Schema.String,
+* }).pipe(Iceberg.identifier("id"));
+* ```
+*/
+const identifier = (...keys) => (schema) => schema.annotate({ icebergIdentifierFields: keys });
+//#endregion
+//#region src/iceberg/index.ts
+var iceberg_exports = /* @__PURE__ */ __exportAll({
+	IcebergSchemaError: () => IcebergSchemaError,
+	identifier: () => identifier,
+	makeCreateTableRequest: () => makeCreateTableRequest
+});
+//#endregion
+export { identifier as n, iceberg_exports as t };
+
+//# sourceMappingURL=iceberg-Cj2PwlNm.mjs.map

package/dist/iceberg-Cj2PwlNm.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"iceberg-Cj2PwlNm.mjs","names":[],"sources":["../src/iceberg/annotations.ts","../src/iceberg/index.ts"],"sourcesContent":["import { Schema } from \"effect\";\n\ndeclare module \"effect/Schema\" {\n  namespace Annotations {\n    interface Annotations {\n      readonly icebergIdentifierFields?: ReadonlyArray<string> | undefined;\n    }\n  }\n}\n\ntype KnownField<S extends Schema.Top> = keyof S[\"Type\"] & string;\n\n/** A row field that cannot be null or undefined. */\nexport type RequiredField<S extends Schema.Top> = [KnownField<S>] extends [never]\n  ? string\n  : {\n      readonly [Field in KnownField<S>]-?: undefined extends S[\"Type\"][Field]\n        ? never\n        : null extends S[\"Type\"][Field]\n          ? never\n          : Field;\n    }[KnownField<S>];\n\n/** A row schema whose identifier fields are `Key`. Only the type has it. */\nexport interface Identified<Key extends string> {\n  readonly \"~iceberg.identifier\": Key;\n}\n\n/**\n * Marks the fields that identify a row. The engine upserts rows by them.\n *\n * Put it last. A later `.annotate()` keeps the runtime annotation, but the type\n * loses the key, so `Resource.entity` stops compiling.\n *\n * @example\n * ```ts\n * const Post = Schema.Struct({\n *   id: Schema.String,\n *   updatedAt: Schema.String,\n * }).pipe(Iceberg.identifier(\"id\"));\n * ```\n */\nexport const identifier =\n  <S extends Schema.Top, const Key extends RequiredField<S>>(\n    ...keys: readonly [Key, ...ReadonlyArray<Key>]\n  ) =>\n  (schema: S): S[\"Rebuild\"] & Identified<Key> =>\n    // Effect has no slot for the key in the schema type, so we add it.\n    schema.annotate({ icebergIdentifierFields: keys }) as S[\"Rebuild\"] & Identified<Key>;\n","export { identifier } from \"./annotations\";\nexport { IcebergSchemaError } from \"./error\";\nexport { makeCreateTableRequest, type CreateTableOptions } from \"./table\";\n"],"mappings":";;;;;;;;;;;;;;;;;;AA0CA,MAAa,cAET,GAAG,UAEJ,WAEC,OAAO,SAAS,EAAE,yBAAyB,KAAK,CAAC"}

package/dist/index-BTAbexBp.d.mts

@@ -0,0 +1,55 @@
+import { r as identifier } from "./annotations-vzSzXtr-.mjs";
+import { Effect, Schema } from "effect";
+import { CreateTableRequest } from "iceberg-js";
+//#region src/iceberg/error.d.ts
+declare const IcebergSchemaError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "IcebergSchemaError";
+} & Readonly<A>;
+/** A schema could not be turned into an Iceberg table. */
+declare class IcebergSchemaError extends IcebergSchemaError_base<{
+  /** Schema path, such as `$.orders.element`. */
+  readonly path: string;
+  readonly message: string;
+  readonly cause?: unknown;
+}> {}
+//#endregion
+//#region src/iceberg/table.d.ts
+/** Options for creating a table. */
+type CreateTableOptions = {
+  readonly name: string;
+  /** Where the table's files go. The catalog may choose this when omitted. */
+  readonly location?: string | undefined;
+  /** Extra table properties. These win over the ones from the schema. */
+  readonly properties?: Readonly<Record<string, string>> | undefined;
+};
+/**
+ * Builds a REST create-table request from an Effect schema.
+ * The catalog assigns the stored field IDs.
+ *
+ * A root `description` becomes the table `comment`, and a field `description`
+ * becomes its `doc`. `title` and `examples` are dropped.
+ *
+ * `Iceberg.identifier` sets the `identifier-field-ids`, and makes every other
+ * top-level column nullable.
+ *
+ * @example
+ * ```ts
+ * const Row = Schema.Struct({
+ *   id: Schema.String.pipe(Schema.annotateKey({ description: "Row identifier" })),
+ * })
+ *   .annotate({ description: "Connector rows" })
+ *   .pipe(Iceberg.identifier("id"))
+ *
+ * const request = yield* Iceberg.makeCreateTableRequest(Row, {
+ *   name: "rows",
+ *   location: "s3://warehouse/rows"
+ * })
+ * ```
+ */
+declare const makeCreateTableRequest: (schema: Schema.Top, options: CreateTableOptions) => Effect.Effect<CreateTableRequest, IcebergSchemaError>;
+declare namespace index_d_exports {
+  export { CreateTableOptions, IcebergSchemaError, identifier, makeCreateTableRequest };
+}
+//#endregion
+export { IcebergSchemaError as i, CreateTableOptions as n, makeCreateTableRequest as r, index_d_exports as t };
+//# sourceMappingURL=index-BTAbexBp.d.mts.map

package/dist/{index-DXa26Ry5.d.ts → index-BX8G7d41.d.mts}

@@ -1,5 +1,4 @@
 import { Duration, Effect, Redacted } from "effect";
-
 //#region src/auth/token-cache.d.ts
 interface ExpiringToken {
   /** The credential kept in memory by the cache. */
@@ -38,4 +37,4 @@ declare namespace index_d_exports {
 }
 //#endregion
 export { makeTokenCache as i, ExpiringToken as n, TokenCache as r, index_d_exports as t };
-//# sourceMappingURL=index-DXa26Ry5.d.ts.map
+//# sourceMappingURL=index-BX8G7d41.d.mts.map

package/dist/{index-VpHaZXZw.d.ts → index-Byr8NTdG.d.mts}

@@ -1,11 +1,9 @@
-import { n as ConnectorDefinition } from "./types-jVD5Ys43.js";
-import { i as StateStore } from "./index-DYpEMJf-.js";
+import { r as ConnectorDefinition } from "./types-NtSv6gwl.mjs";
+import { i as StateStore } from "./index-CgvFomO6.mjs";
 import { Schema } from "effect";
-import { HttpRouter } from "effect/unstable/http";
-import * as effect_Layer0 from "effect/Layer";
-
+import { HttpRouter } from "effect/http";
 //#region src/status/route.d.ts
-declare const statusRoute: (connector: ConnectorDefinition, path?: HttpRouter.PathInput) => effect_Layer0.Layer<never, never, HttpRouter.HttpRouter | HttpRouter.Request<"Requires", StateStore>>;
+declare const statusRoute: (connector: ConnectorDefinition, path?: HttpRouter.PathInput) => import("effect/Layer").Layer<never, never, HttpRouter.HttpRouter | HttpRouter.Request<"Requires", StateStore>>;
 //#endregion
 //#region src/status/types.d.ts
 declare const SyncStateSchema: Schema.Literals<readonly ["pending", "backfilling", "live", "error"]>;
@@ -14,8 +12,8 @@ type SyncState = Schema.Schema.Type<typeof SyncStateSchema>;
 declare const ResourceErrorSchema: Schema.Struct<{
   readonly at: Schema.String;
   readonly source: Schema.Literals<readonly ["backfill", "changes"]>;
-  readonly operation: Schema.Literals<readonly ["fetch", "publish", "checkpoint"]>;
-  readonly code: Schema.Literals<readonly ["fetch_failed", "publish_failed", "checkpoint_failed"]>;
+  readonly operation: Schema.Literals<readonly ["fetch", "ingest", "checkpoint"]>;
+  readonly code: Schema.Literals<readonly ["fetch_failed", "ingest_failed", "checkpoint_failed"]>;
   readonly message: Schema.String;
 }>;
 type ResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;
@@ -33,8 +31,8 @@ declare const ResourceStatusSchema: Schema.Struct<{
   readonly lastError: Schema.optional<Schema.Struct<{
     readonly at: Schema.String;
     readonly source: Schema.Literals<readonly ["backfill", "changes"]>;
-    readonly operation: Schema.Literals<readonly ["fetch", "publish", "checkpoint"]>;
-    readonly code: Schema.Literals<readonly ["fetch_failed", "publish_failed", "checkpoint_failed"]>;
+    readonly operation: Schema.Literals<readonly ["fetch", "ingest", "checkpoint"]>;
+    readonly code: Schema.Literals<readonly ["fetch_failed", "ingest_failed", "checkpoint_failed"]>;
     readonly message: Schema.String;
   }>>;
   readonly name: Schema.String;
@@ -57,8 +55,8 @@ declare const StatusResponseSchema: Schema.Struct<{
     readonly lastError: Schema.optional<Schema.Struct<{
       readonly at: Schema.String;
       readonly source: Schema.Literals<readonly ["backfill", "changes"]>;
-      readonly operation: Schema.Literals<readonly ["fetch", "publish", "checkpoint"]>;
-      readonly code: Schema.Literals<readonly ["fetch_failed", "publish_failed", "checkpoint_failed"]>;
+      readonly operation: Schema.Literals<readonly ["fetch", "ingest", "checkpoint"]>;
+      readonly code: Schema.Literals<readonly ["fetch_failed", "ingest_failed", "checkpoint_failed"]>;
       readonly message: Schema.String;
     }>>;
     readonly name: Schema.String;
@@ -77,4 +75,4 @@ declare namespace index_d_exports {
 }
 //#endregion
 export { ResourceStatusSchema as a, StatusUnavailableResponse as c, SyncStateSchema as d, statusRoute as f, ResourceStatus as i, StatusUnavailableResponseSchema as l, ResourceError as n, StatusResponse as o, ResourceErrorSchema as r, StatusResponseSchema as s, index_d_exports as t, SyncState as u };
-//# sourceMappingURL=index-VpHaZXZw.d.ts.map
+//# sourceMappingURL=index-Byr8NTdG.d.mts.map