@useairfoil/connector-kit 0.6.1 → 0.7.0

package/dist/arrow-flight-DDQ0ktx9.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-flight-DDQ0ktx9.mjs","names":["Struct","ArrowSchema","RuntimeConfig.catalog","RuntimeConfig.engineUri","RuntimeConfig.flightUri","flightUri","RuntimeConfig.loadTableBindings"],"sources":["../src/ingestor/arrow.ts","../src/ingestor/arrow-flight.ts"],"sourcesContent":["import {\n  Bool,\n  DateDay,\n  Decimal,\n  Field,\n  FixedSizeBinary,\n  Float32,\n  Float64,\n  Int32,\n  Int64,\n  LargeBinary,\n  List,\n  Map_,\n  RecordBatch,\n  Schema as ArrowSchema,\n  Struct,\n  TimeMicrosecond,\n  TimestampMicrosecond,\n  Utf8,\n  makeBuilder,\n  type DataType,\n  type TypeMap,\n} from \"apache-arrow\";\nimport { Effect, Schema } from \"effect\";\nimport {\n  parseDecimalType,\n  parseFixedType,\n  type IcebergType,\n  type PrimitiveType,\n  type TableSchema,\n} from \"iceberg-js\";\n\nimport { ConnectorError } from \"../errors\";\n\n// Nothing converts an Iceberg schema to Arrow for us, so we do it here.\n// Each type also gets a schema for its values, because Arrow does not reject a bad\n// value, it converts it: \"1\" becomes 1, and 1.5 becomes 1.\n\ntype RowSchema = Schema.Codec<unknown, unknown>;\n\n/** One Iceberg type: the Arrow type, and the values it accepts. */\ntype Compiled = {\n  readonly type: DataType;\n  readonly row: RowSchema;\n};\n\n// Arrow pads or cuts binary values to fit.\nconst bytes = (length: number) =>\n  Schema.Uint8Array.check(\n    Schema.makeFilter(\n      (value: Uint8Array) => value.byteLength === length || `expected ${length} bytes`,\n    ),\n  );\n\n// Arrow wraps big integers instead of failing, so check the range.\nconst int32 = Schema.Number.check(\n  Schema.isInt(),\n  Schema.isBetween({ minimum: -2_147_483_648, maximum: 2_147_483_647 }),\n);\n\nconst int64 = Schema.BigInt.check(\n  Schema.isBetweenBigInt({ minimum: -(2n ** 63n), maximum: 2n ** 63n - 1n }),\n);\n\n// Arrow turns a number outside the 32-bit range into Infinity.\nconst float32 = Schema.Finite.check(\n  Schema.makeFilter(\n    (value: number) => Number.isFinite(Math.fround(value)) || \"expected a 32-bit number\",\n  ),\n);\n\nconst microsecondsInDay = Schema.BigInt.check(\n  Schema.isBetweenBigInt({ minimum: 0n, maximum: 86_399_999_999n }),\n);\n\nconst primitives: Record<string, Compiled> = {\n  boolean: { type: new Bool(), row: Schema.Boolean },\n  int: { type: new Int32(), row: int32 },\n  long: { type: new Int64(), row: int64 },\n  float: { type: new Float32(), row: float32 },\n  double: { type: new Float64(), row: Schema.Finite },\n  string: { type: new Utf8(), row: Schema.String },\n  uuid: { type: new FixedSizeBinary(16), row: bytes(16) },\n  binary: { type: new LargeBinary(), row: Schema.Uint8Array },\n  date: { type: new DateDay(), row: Schema.Date },\n  time: { type: new TimeMicrosecond(), row: microsecondsInDay },\n  timestamp: { type: new TimestampMicrosecond(), row: Schema.Date },\n  timestamptz: { type: new TimestampMicrosecond(\"UTC\"), row: Schema.Date },\n};\n\nconst primitive = (type: PrimitiveType): Compiled => {\n  const known = primitives[type];\n\n  if (known) return known;\n\n  // Nothing checks the catalog metadata, and Arrow takes any precision.\n  // Better to fail here than at write time.\n  const decimal = parseDecimalType(type);\n\n  if (decimal) {\n    if (decimal.precision < 1 || decimal.precision > 38 || decimal.scale > decimal.precision) {\n      throw new Error(`Invalid Iceberg decimal type ${type}`);\n    }\n\n    return {\n      type: new Decimal(decimal.scale, decimal.precision),\n      row: Schema.instanceOf(Uint32Array).check(\n        Schema.makeFilter((value: Uint32Array) => value.length === 4 || \"expected four words\"),\n      ),\n    };\n  }\n\n  const fixed = parseFixedType(type);\n\n  if (fixed) {\n    if (fixed.length < 1) throw new Error(`Invalid Iceberg fixed type ${type}`);\n\n    return { type: new FixedSizeBinary(fixed.length), row: bytes(fixed.length) };\n  }\n\n  throw new Error(`Unsupported Iceberg type ${type}`);\n};\n\n/** Absent and null both become null in Arrow. */\nconst optional = (row: RowSchema) => Schema.optional(Schema.NullOr(row));\n\nconst fieldMetadata = (id: number): Map<string, string> =>\n  new Map([[\"PARQUET:field_id\", String(id)]]);\n\n/** Same as `primitive`, plus struct, list and map. */\nconst compileType = (type: IcebergType): Compiled => {\n  if (typeof type === \"string\") return primitive(type);\n\n  if (type.type === \"struct\") {\n    const fields = type.fields.map((field) => ({ ...field, compiled: compileType(field.type) }));\n\n    return {\n      type: new Struct(\n        fields.map(\n          (field) =>\n            new Field(field.name, field.compiled.type, !field.required, fieldMetadata(field.id)),\n        ),\n      ),\n      row: Schema.Struct(\n        Object.fromEntries(\n          fields.map((field) => [\n            field.name,\n            field.required ? field.compiled.row : optional(field.compiled.row),\n          ]),\n        ),\n      ),\n    };\n  }\n\n  if (type.type === \"list\") {\n    const element = compileType(type.element);\n    const required = type[\"element-required\"];\n\n    return {\n      type: new List(\n        new Field(\"element\", element.type, !required, fieldMetadata(type[\"element-id\"])),\n      ),\n      row: Schema.Array(required ? element.row : Schema.NullishOr(element.row)),\n    };\n  }\n\n  const key = compileType(type.key);\n  const value = compileType(type.value);\n  const required = type[\"value-required\"];\n  const entries = new Struct([\n    new Field(\"key\", key.type, false, fieldMetadata(type[\"key-id\"])),\n    new Field(\"value\", value.type, !required, fieldMetadata(type[\"value-id\"])),\n  ]);\n\n  return {\n    type: new Map_(new Field(\"entries\", entries, false)),\n    row: Schema.ReadonlyMap(key.row, required ? value.row : Schema.NullishOr(value.row)),\n  };\n};\n\nexport type RowEncoder = (\n  rows: ReadonlyArray<object>,\n) => Effect.Effect<RecordBatch<TypeMap>, ConnectorError>;\n\nexport const makeRowEncoder = (\n  tableSchema: TableSchema,\n  version: string,\n): Effect.Effect<RowEncoder, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      const fields = tableSchema.fields.map((field) => ({\n        id: field.id,\n        name: field.name,\n        ...compileType(field.type),\n      }));\n      const fieldNames = new Set(fields.map((field) => field.name));\n      const identifierIds = new Set(tableSchema[\"identifier-field-ids\"]);\n      const keys = tableSchema.fields\n        .filter((field) => identifierIds.has(field.id))\n        .map((field) => field.name);\n      // Other columns may be missing: a delete only sends the key and version.\n      const required = new Set([...keys, version]);\n\n      if (!fieldNames.has(version)) throw new Error(`Missing version column ${version}`);\n\n      const struct = new Struct(\n        fields.map((field) => new Field(field.name, field.type, true, fieldMetadata(field.id))),\n      );\n      const schema = new ArrowSchema(struct.children);\n\n      const decodeRow = Schema.decodeUnknownSync(\n        Schema.Struct(\n          Object.fromEntries(\n            fields.map((field) => [\n              field.name,\n              required.has(field.name) ? field.row : optional(field.row),\n            ]),\n          ),\n        ),\n        { onExcessProperty: \"error\" },\n      );\n\n      // Schema says which field is wrong. This adds the row number.\n      const decodeAt = (row: object, index: number) => {\n        try {\n          return decodeRow(row) as Record<string, unknown>;\n        } catch (cause) {\n          const detail = cause instanceof Error ? cause.message : String(cause);\n          throw new Error(`row ${index}: ${detail}`, { cause });\n        }\n      };\n\n      return (rows: ReadonlyArray<object>) =>\n        Effect.try({\n          try: () => {\n            const builder = makeBuilder({ type: struct, nullValues: [null, undefined] });\n\n            for (const [index, row] of rows.entries()) {\n              builder.append(decodeAt(row, index) as Struct<TypeMap>[\"TValue\"]);\n            }\n\n            return new RecordBatch(schema, builder.finish().flush());\n          },\n          catch: (cause) => new ConnectorError({ message: \"Failed to encode rows\", cause }),\n        });\n    },\n    catch: (cause) => new ConnectorError({ message: \"Failed to convert Iceberg schema\", cause }),\n  });\n","import type { RecordBatch, TypeMap } from \"apache-arrow\";\nimport type { TableIdentifier } from \"iceberg-js\";\n\nimport { CatalogManager } from \"@useairfoil/api\";\nimport { Config, Effect, Layer } from \"effect\";\nimport { FetchHttpClient } from \"effect/http\";\n\nimport type { TableSource } from \"../core/types\";\n\nimport { ConnectorError } from \"../errors\";\nimport { compileTable } from \"../iceberg/table\";\nimport * as RuntimeConfig from \"../runtime-config\";\nimport { makeRowEncoder, type RowEncoder } from \"./arrow\";\nimport { Ingestor, type IngestorService } from \"./service\";\n\n/** One catalog and an Iceberg table for each resource that runs. */\nexport type ArrowFlightIngestorConfig<Source extends TableSource = TableSource> = {\n  readonly connector: Source;\n  readonly catalog: string;\n  readonly tables: RuntimeConfig.ResolvedTableBindings<Source>;\n};\n\ntype IngestorEntry = {\n  readonly encode: RowEncoder;\n  readonly push: (batch: RecordBatch<TypeMap>) => Effect.Effect<void, ConnectorError>;\n};\n\ntype PreparedTable = {\n  readonly resource: string;\n  readonly identifier: TableIdentifier;\n  readonly encode: RowEncoder;\n};\n\nconst connectorError = (message: string, cause?: unknown) => new ConnectorError({ message, cause });\n\n/** Compiles local schemas and opens one Arrow Flight stream per bound resource. */\nexport const layerArrowFlight = <const Source extends TableSource>(\n  config: ArrowFlightIngestorConfig<Source>,\n): Layer.Layer<Ingestor, ConnectorError, CatalogManager.CatalogManager> =>\n  Layer.effect(Ingestor)(\n    Effect.gen(function* () {\n      const manager = yield* CatalogManager.CatalogManager;\n      const bindings: Partial<Record<string, RuntimeConfig.TableBinding>> = config.tables;\n\n      const prepared: Array<PreparedTable> = [];\n      for (const resource of config.connector.resources) {\n        const binding = bindings[resource.name];\n        if (!binding) continue;\n\n        const { namespace, name } = binding;\n        const identifier = { namespace, name };\n        const { schema } = yield* compileTable(resource.rowSchema).pipe(\n          Effect.mapError((cause) =>\n            connectorError(`Invalid schema for ${resource.name}: ${cause.message}`, cause),\n          ),\n        );\n\n        const encode = yield* makeRowEncoder(schema, resource.version);\n        prepared.push({ resource: resource.name, identifier, encode });\n      }\n\n      const entries = new Map<string, IngestorEntry>();\n      for (const { resource, identifier, encode } of prepared) {\n        const ingestor = yield* manager\n          .ingestor({\n            catalog: config.catalog,\n            namespace: identifier.namespace,\n            table: identifier.name,\n          })\n          .pipe(\n            Effect.mapError((cause) =>\n              connectorError(`Failed to open ingestor for ${resource}`, cause),\n            ),\n          );\n\n        entries.set(resource, {\n          encode,\n          push: (batch) =>\n            ingestor\n              .push(batch)\n              .pipe(\n                Effect.mapError((cause) =>\n                  connectorError(`Failed to ingest rows for ${resource}`, cause),\n                ),\n              ),\n        });\n      }\n\n      const service: IngestorService = {\n        resourceNames: prepared.map(({ resource }) => resource),\n        ingest: ({ resource, batch }) => {\n          const entry = entries.get(resource);\n\n          if (!entry) return Effect.fail(connectorError(`Unknown resource ${resource}`));\n          if (batch.rows.length === 0) return Effect.succeed({ rows: 0, bytes: 0 });\n\n          return entry\n            .encode(batch.rows)\n            .pipe(\n              Effect.flatMap((encoded) =>\n                entry\n                  .push(encoded)\n                  .pipe(Effect.as({ rows: batch.rows.length, bytes: encoded.data.byteLength })),\n              ),\n            );\n        },\n      };\n\n      return Ingestor.of(service);\n    }),\n  );\n\n/** Configures Arrow Flight ingestion from the hosted URL, catalog and table bindings. */\nexport const layerArrowFlightConfig = (connector: TableSource) =>\n  Layer.unwrap(\n    Effect.gen(function* () {\n      const { catalog, uri, flightUri } = yield* Config.all({\n        catalog: RuntimeConfig.catalog,\n        uri: RuntimeConfig.engineUri,\n        flightUri: RuntimeConfig.flightUri,\n      }).pipe(\n        Effect.mapError((cause) => connectorError(\"Invalid Arrow Flight configuration\", cause)),\n      );\n\n      if (uri.protocol !== \"http:\" && uri.protocol !== \"https:\") {\n        return yield* Effect.fail(connectorError(\"Invalid Arrow Flight URL\"));\n      }\n      if (flightUri.protocol !== \"http:\" && flightUri.protocol !== \"https:\") {\n        return yield* Effect.fail(connectorError(\"Invalid Arrow Flight URL\"));\n      }\n\n      const tables = yield* RuntimeConfig.loadTableBindings(connector).pipe(\n        Effect.mapError((cause) => connectorError(cause.message, cause)),\n      );\n      const manager = CatalogManager.layer({ engineUri: uri.href, flightUri: flightUri.href }).pipe(\n        Layer.provide(FetchHttpClient.layer),\n      );\n\n      return layerArrowFlight({ connector, catalog, tables }).pipe(Layer.provide(manager));\n    }),\n  );\n"],"mappings":";;;;;;;;;;AA+CA,MAAM,SAAS,WACb,OAAO,WAAW,MAChB,OAAO,YACJ,UAAsB,MAAM,eAAe,UAAU,YAAY,OAAO,OAC3E,CACF;AAGF,MAAM,QAAQ,OAAO,OAAO,MAC1B,OAAO,MAAM,GACb,OAAO,UAAU;CAAE,SAAS;CAAgB,SAAS;AAAc,CAAC,CACtE;AAEA,MAAM,QAAQ,OAAO,OAAO,MAC1B,OAAO,gBAAgB;CAAE,SAAS,EAAE,MAAM;CAAM,SAAS,MAAM,MAAM;AAAG,CAAC,CAC3E;AAGA,MAAM,UAAU,OAAO,OAAO,MAC5B,OAAO,YACJ,UAAkB,OAAO,SAAS,KAAK,OAAO,KAAK,CAAC,KAAK,0BAC5D,CACF;AAEA,MAAM,oBAAoB,OAAO,OAAO,MACtC,OAAO,gBAAgB;CAAE,SAAS;CAAI,SAAS;AAAgB,CAAC,CAClE;AAEA,MAAM,aAAuC;CAC3C,SAAS;EAAE,MAAM,IAAI,KAAK;EAAG,KAAK,OAAO;CAAQ;CACjD,KAAK;EAAE,MAAM,IAAI,MAAM;EAAG,KAAK;CAAM;CACrC,MAAM;EAAE,MAAM,IAAI,MAAM;EAAG,KAAK;CAAM;CACtC,OAAO;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK;CAAQ;CAC3C,QAAQ;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK,OAAO;CAAO;CAClD,QAAQ;EAAE,MAAM,IAAI,KAAK;EAAG,KAAK,OAAO;CAAO;CAC/C,MAAM;EAAE,MAAM,IAAI,gBAAgB,EAAE;EAAG,KAAK,MAAM,EAAE;CAAE;CACtD,QAAQ;EAAE,MAAM,IAAI,YAAY;EAAG,KAAK,OAAO;CAAW;CAC1D,MAAM;EAAE,MAAM,IAAI,QAAQ;EAAG,KAAK,OAAO;CAAK;CAC9C,MAAM;EAAE,MAAM,IAAI,gBAAgB;EAAG,KAAK;CAAkB;CAC5D,WAAW;EAAE,MAAM,IAAI,qBAAqB;EAAG,KAAK,OAAO;CAAK;CAChE,aAAa;EAAE,MAAM,IAAI,qBAAqB,KAAK;EAAG,KAAK,OAAO;CAAK;AACzE;AAEA,MAAM,aAAa,SAAkC;CACnD,MAAM,QAAQ,WAAW;CAEzB,IAAI,OAAO,OAAO;CAIlB,MAAM,UAAU,iBAAiB,IAAI;CAErC,IAAI,SAAS;EACX,IAAI,QAAQ,YAAY,KAAK,QAAQ,YAAY,MAAM,QAAQ,QAAQ,QAAQ,WAC7E,MAAM,IAAI,MAAM,gCAAgC,MAAM;EAGxD,OAAO;GACL,MAAM,IAAI,QAAQ,QAAQ,OAAO,QAAQ,SAAS;GAClD,KAAK,OAAO,WAAW,WAAW,CAAC,CAAC,MAClC,OAAO,YAAY,UAAuB,MAAM,WAAW,KAAK,qBAAqB,CACvF;EACF;CACF;CAEA,MAAM,QAAQ,eAAe,IAAI;CAEjC,IAAI,OAAO;EACT,IAAI,MAAM,SAAS,GAAG,MAAM,IAAI,MAAM,8BAA8B,MAAM;EAE1E,OAAO;GAAE,MAAM,IAAI,gBAAgB,MAAM,MAAM;GAAG,KAAK,MAAM,MAAM,MAAM;EAAE;CAC7E;CAEA,MAAM,IAAI,MAAM,4BAA4B,MAAM;AACpD;;AAGA,MAAM,YAAY,QAAmB,OAAO,SAAS,OAAO,OAAO,GAAG,CAAC;AAEvE,MAAM,iBAAiB,uBACrB,IAAI,IAAI,CAAC,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC,CAAC;;AAG5C,MAAM,eAAe,SAAgC;CACnD,IAAI,OAAO,SAAS,UAAU,OAAO,UAAU,IAAI;CAEnD,IAAI,KAAK,SAAS,UAAU;EAC1B,MAAM,SAAS,KAAK,OAAO,KAAK,WAAW;GAAE,GAAG;GAAO,UAAU,YAAY,MAAM,IAAI;EAAE,EAAE;EAE3F,OAAO;GACL,MAAM,IAAIA,SACR,OAAO,KACJ,UACC,IAAI,MAAM,MAAM,MAAM,MAAM,SAAS,MAAM,CAAC,MAAM,UAAU,cAAc,MAAM,EAAE,CAAC,CACvF,CACF;GACA,KAAK,OAAO,OACV,OAAO,YACL,OAAO,KAAK,UAAU,CACpB,MAAM,MACN,MAAM,WAAW,MAAM,SAAS,MAAM,SAAS,MAAM,SAAS,GAAG,CACnE,CAAC,CACH,CACF;EACF;CACF;CAEA,IAAI,KAAK,SAAS,QAAQ;EACxB,MAAM,UAAU,YAAY,KAAK,OAAO;EACxC,MAAM,WAAW,KAAK;EAEtB,OAAO;GACL,MAAM,IAAI,KACR,IAAI,MAAM,WAAW,QAAQ,MAAM,CAAC,UAAU,cAAc,KAAK,aAAa,CAAC,CACjF;GACA,KAAK,OAAO,MAAM,WAAW,QAAQ,MAAM,OAAO,UAAU,QAAQ,GAAG,CAAC;EAC1E;CACF;CAEA,MAAM,MAAM,YAAY,KAAK,GAAG;CAChC,MAAM,QAAQ,YAAY,KAAK,KAAK;CACpC,MAAM,WAAW,KAAK;CACtB,MAAM,UAAU,IAAIA,SAAO,CACzB,IAAI,MAAM,OAAO,IAAI,MAAM,OAAO,cAAc,KAAK,SAAS,CAAC,GAC/D,IAAI,MAAM,SAAS,MAAM,MAAM,CAAC,UAAU,cAAc,KAAK,WAAW,CAAC,CAC3E,CAAC;CAED,OAAO;EACL,MAAM,IAAI,KAAK,IAAI,MAAM,WAAW,SAAS,KAAK,CAAC;EACnD,KAAK,OAAO,YAAY,IAAI,KAAK,WAAW,MAAM,MAAM,OAAO,UAAU,MAAM,GAAG,CAAC;CACrF;AACF;AAMA,MAAa,kBACX,aACA,YAEA,OAAO,IAAI;CACT,WAAW;EACT,MAAM,SAAS,YAAY,OAAO,KAAK,WAAW;GAChD,IAAI,MAAM;GACV,MAAM,MAAM;GACZ,GAAG,YAAY,MAAM,IAAI;EAC3B,EAAE;EACF,MAAM,aAAa,IAAI,IAAI,OAAO,KAAK,UAAU,MAAM,IAAI,CAAC;EAC5D,MAAM,gBAAgB,IAAI,IAAI,YAAY,uBAAuB;EACjE,MAAM,OAAO,YAAY,OACtB,QAAQ,UAAU,cAAc,IAAI,MAAM,EAAE,CAAC,CAAC,CAC9C,KAAK,UAAU,MAAM,IAAI;EAE5B,MAAM,2BAAW,IAAI,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC;EAE3C,IAAI,CAAC,WAAW,IAAI,OAAO,GAAG,MAAM,IAAI,MAAM,0BAA0B,SAAS;EAEjF,MAAM,SAAS,IAAIA,SACjB,OAAO,KAAK,UAAU,IAAI,MAAM,MAAM,MAAM,MAAM,MAAM,MAAM,cAAc,MAAM,EAAE,CAAC,CAAC,CACxF;EACA,MAAM,SAAS,IAAIC,SAAY,OAAO,QAAQ;EAE9C,MAAM,YAAY,OAAO,kBACvB,OAAO,OACL,OAAO,YACL,OAAO,KAAK,UAAU,CACpB,MAAM,MACN,SAAS,IAAI,MAAM,IAAI,IAAI,MAAM,MAAM,SAAS,MAAM,GAAG,CAC3D,CAAC,CACH,CACF,GACA,EAAE,kBAAkB,QAAQ,CAC9B;EAGA,MAAM,YAAY,KAAa,UAAkB;GAC/C,IAAI;IACF,OAAO,UAAU,GAAG;GACtB,SAAS,OAAO;IACd,MAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;IACpE,MAAM,IAAI,MAAM,OAAO,MAAM,IAAI,UAAU,EAAE,MAAM,CAAC;GACtD;EACF;EAEA,QAAQ,SACN,OAAO,IAAI;GACT,WAAW;IACT,MAAM,UAAU,YAAY;KAAE,MAAM;KAAQ,YAAY,CAAC,MAAM,KAAA,CAAS;IAAE,CAAC;IAE3E,KAAK,MAAM,CAAC,OAAO,QAAQ,KAAK,QAAQ,GACtC,QAAQ,OAAO,SAAS,KAAK,KAAK,CAA8B;IAGlE,OAAO,IAAI,YAAY,QAAQ,QAAQ,OAAO,CAAC,CAAC,MAAM,CAAC;GACzD;GACA,QAAQ,UAAU,IAAI,eAAe;IAAE,SAAS;IAAyB;GAAM,CAAC;EAClF,CAAC;CACL;CACA,QAAQ,UAAU,IAAI,eAAe;EAAE,SAAS;EAAoC;CAAM,CAAC;AAC7F,CAAC;;;ACtNH,MAAM,kBAAkB,SAAiB,UAAoB,IAAI,eAAe;CAAE;CAAS;AAAM,CAAC;;AAGlG,MAAa,oBACX,WAEA,MAAM,OAAO,QAAQ,CAAC,CACpB,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,eAAe;CACtC,MAAM,WAAgE,OAAO;CAE7E,MAAM,WAAiC,CAAC;CACxC,KAAK,MAAM,YAAY,OAAO,UAAU,WAAW;EACjD,MAAM,UAAU,SAAS,SAAS;EAClC,IAAI,CAAC,SAAS;EAEd,MAAM,EAAE,WAAW,SAAS;EAC5B,MAAM,aAAa;GAAE;GAAW;EAAK;EACrC,MAAM,EAAE,WAAW,OAAO,aAAa,SAAS,SAAS,CAAC,CAAC,KACzD,OAAO,UAAU,UACf,eAAe,sBAAsB,SAAS,KAAK,IAAI,MAAM,WAAW,KAAK,CAC/E,CACF;EAEA,MAAM,SAAS,OAAO,eAAe,QAAQ,SAAS,OAAO;EAC7D,SAAS,KAAK;GAAE,UAAU,SAAS;GAAM;GAAY;EAAO,CAAC;CAC/D;CAEA,MAAM,0BAAU,IAAI,IAA2B;CAC/C,KAAK,MAAM,EAAE,UAAU,YAAY,YAAY,UAAU;EACvD,MAAM,WAAW,OAAO,QACrB,SAAS;GACR,SAAS,OAAO;GAChB,WAAW,WAAW;GACtB,OAAO,WAAW;EACpB,CAAC,CAAC,CACD,KACC,OAAO,UAAU,UACf,eAAe,+BAA+B,YAAY,KAAK,CACjE,CACF;EAEF,QAAQ,IAAI,UAAU;GACpB;GACA,OAAO,UACL,SACG,KAAK,KAAK,CAAC,CACX,KACC,OAAO,UAAU,UACf,eAAe,6BAA6B,YAAY,KAAK,CAC/D,CACF;EACN,CAAC;CACH;CAEA,MAAM,UAA2B;EAC/B,eAAe,SAAS,KAAK,EAAE,eAAe,QAAQ;EACtD,SAAS,EAAE,UAAU,YAAY;GAC/B,MAAM,QAAQ,QAAQ,IAAI,QAAQ;GAElC,IAAI,CAAC,OAAO,OAAO,OAAO,KAAK,eAAe,oBAAoB,UAAU,CAAC;GAC7E,IAAI,MAAM,KAAK,WAAW,GAAG,OAAO,OAAO,QAAQ;IAAE,MAAM;IAAG,OAAO;GAAE,CAAC;GAExE,OAAO,MACJ,OAAO,MAAM,IAAI,CAAC,CAClB,KACC,OAAO,SAAS,YACd,MACG,KAAK,OAAO,CAAC,CACb,KAAK,OAAO,GAAG;IAAE,MAAM,MAAM,KAAK;IAAQ,OAAO,QAAQ,KAAK;GAAW,CAAC,CAAC,CAChF,CACF;EACJ;CACF;CAEA,OAAO,SAAS,GAAG,OAAO;AAC5B,CAAC,CACH;;AAGF,MAAa,0BAA0B,cACrC,MAAM,OACJ,OAAO,IAAI,aAAa;CACtB,MAAM,EAAE,SAAA,WAAS,KAAK,WAAA,gBAAc,OAAO,OAAO,IAAI;EAC3CC;EACT,KAAKC;EACMC;CACb,CAAC,CAAC,CAAC,KACD,OAAO,UAAU,UAAU,eAAe,sCAAsC,KAAK,CAAC,CACxF;CAEA,IAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAC/C,OAAO,OAAO,OAAO,KAAK,eAAe,0BAA0B,CAAC;CAEtE,IAAIC,YAAU,aAAa,WAAWA,YAAU,aAAa,UAC3D,OAAO,OAAO,OAAO,KAAK,eAAe,0BAA0B,CAAC;CAGtE,MAAM,SAAS,OAAOC,kBAAgC,SAAS,CAAC,CAAC,KAC/D,OAAO,UAAU,UAAU,eAAe,MAAM,SAAS,KAAK,CAAC,CACjE;CACA,MAAM,UAAU,eAAe,MAAM;EAAE,WAAW,IAAI;EAAM,WAAWD,YAAU;CAAK,CAAC,CAAC,CAAC,KACvF,MAAM,QAAQ,gBAAgB,KAAK,CACrC;CAEA,OAAO,iBAAiB;EAAE;EAAW,SAAA;EAAS;CAAO,CAAC,CAAC,CAAC,KAAK,MAAM,QAAQ,OAAO,CAAC;AACrF,CAAC,CACH"}

package/dist/connector-app/index.d.mts

@@ -1,2 +1,2 @@
-import { a as makeDemoCommand, g as check, h as ResourceCheckResult, i as start, m as CheckResult, n as App, o as parseSize, p as CheckOptions, r as StartOptions, s as makeCreateTableCommand } from "../index-NnCutIqM.mjs";
-export { type App, type CheckOptions, type CheckResult, type ResourceCheckResult, type StartOptions, check, makeCreateTableCommand, makeDemoCommand, parseSize, start };
+import { _ as check, a as start, c as makeCreateTableCommand, g as ResourceCheckResult, h as CheckResult, i as StartOptions, m as CheckOptions, n as makeSetupCommand, o as makeDemoCommand, r as App, s as parseSize } from "../index-CZKx2Yce.mjs";
+export { type App, type CheckOptions, type CheckResult, type ResourceCheckResult, type StartOptions, check, makeCreateTableCommand, makeDemoCommand, makeSetupCommand, parseSize, start };

package/dist/connector-app/index.mjs

@@ -1,2 +1,2 @@
-import { a as makeCreateTableCommand, i as parseSize, n as start, o as check, r as makeDemoCommand } from "../connector-app-C-L4QqG6.mjs";
-export { check, makeCreateTableCommand, makeDemoCommand, parseSize, start };
+import { a as parseSize, i as makeDemoCommand, n as makeSetupCommand, o as makeCreateTableCommand, r as start, s as check } from "../connector-app-C6_dZnhE.mjs";
+export { check, makeCreateTableCommand, makeDemoCommand, makeSetupCommand, parseSize, start };

package/dist/connector-app-C6_dZnhE.mjs

@@ -0,0 +1,256 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
+import { a as IcebergSchemaError, n as compileTable, r as makeCreateRequest } from "./table-CxM09xf7.mjs";
+import { a as catalog, n as loadTableBindings, o as engineUri } from "./runtime-config-Bn8Fivt1.mjs";
+import { n as run } from "./demo-DqTJ7TWI.mjs";
+import { n as layerArrowFlightConfig } from "./arrow-flight-DDQ0ktx9.mjs";
+import { n as run$1 } from "./ingestion-DoCUwm5N.mjs";
+import { DateTime, Duration, Effect, Option, Schedule, Schema, Tuple } from "effect";
+import { Argument, CliError, Command, Flag } from "effect/cli";
+import { CatalogManager } from "@useairfoil/api";
+import "effect/http";
+import { getCurrentSchema } from "iceberg-js";
+import humanFormat from "human-format";
+import { NodeHttpServer } from "@effect/platform-node";
+import { createServer } from "node:http";
+//#region src/connector-app/check.ts
+const ok = { _tag: "ok" };
+const error = (message) => ({
+	_tag: "error",
+	message
+});
+const fromEntries = (entries) => Object.fromEntries(entries);
+const resultsFor = (names, result) => fromEntries(names.map((name) => Tuple.make(name, result)));
+/**
+* Builds the connector service from its layer and validates only the selected
+* resources. The layer remains open until every check finishes.
+*/
+function check(service, layer, options) {
+	if (options.resources.length === 0) return Effect.succeed(fromEntries([]));
+	return service.pipe(Effect.flatMap((connector) => Effect.forEach(options.resources, (name) => {
+		const resource = connector.resources.find((candidate) => candidate.name === name);
+		return (resource ? resource.check.pipe(Effect.match({
+			onFailure: (failure) => error(failure.message),
+			onSuccess: () => ok
+		})) : Effect.succeed(error(`Unknown connector resource: ${name}`))).pipe(Effect.map((result) => Tuple.make(name, result)));
+	}).pipe(Effect.map(fromEntries))), Effect.provide(layer), Effect.match({
+		onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
+		onSuccess: (results) => results
+	}));
+}
+//#endregion
+//#region src/catalog/retry.ts
+/** Retries transient catalog responses at the request boundary. */
+const retryCatalogRequest = (request, options) => options === void 0 ? request : request.pipe(Effect.retry({
+	schedule: Schedule.spaced(options.interval),
+	times: options.times,
+	while: (error) => error.status === 408 || error.status === 429 || error.status !== void 0 && error.status >= 500 && error.status <= 599
+}));
+//#endregion
+//#region src/catalog/table.ts
+const tableError = (cause) => {
+	if (cause instanceof ConnectorError) return cause;
+	const message = cause instanceof Error ? cause.message : "Failed to prepare Iceberg table";
+	return new ConnectorError({
+		message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,
+		cause
+	});
+};
+const compareTypes = (expected, actual, path) => {
+	if (typeof expected === "string" || typeof actual === "string") return expected === actual ? void 0 : path;
+	if (expected.type !== actual.type) return path;
+	switch (expected.type) {
+		case "struct":
+			if (actual.type !== "struct") return path;
+			return compareFields(expected.fields, actual.fields, path);
+		case "list":
+			if (actual.type !== "list" || expected["element-required"] !== actual["element-required"]) return path;
+			return compareTypes(expected.element, actual.element, `${path}.element`);
+		case "map":
+			if (actual.type !== "map" || expected["value-required"] !== actual["value-required"]) return path;
+			return compareTypes(expected.key, actual.key, `${path}.key`) ?? compareTypes(expected.value, actual.value, `${path}.value`);
+	}
+};
+const compareFields = (expected, actual, path) => {
+	if (expected.length !== actual.length) return path;
+	for (const field of expected) {
+		const fieldPath = `${path}.${field.name}`;
+		const other = actual.find((candidate) => candidate.name === field.name);
+		if (!other || field.required !== other.required) return fieldPath;
+		const mismatch = compareTypes(field.type, other.type, fieldPath);
+		if (mismatch) return mismatch;
+	}
+};
+const verifyTable = (metadata, expected) => Effect.try({
+	try: () => {
+		if (!Array.isArray(metadata.schemas)) throw new Error("Invalid table metadata");
+		if (metadata["format-version"] !== 3) throw new Error("Storage-engine tables must use Iceberg format v3. Migrate the table before ingesting.");
+		const partitionSpec = metadata["partition-specs"].find((spec) => spec["spec-id"] === metadata["default-spec-id"]);
+		if (!partitionSpec) throw new Error("Table has no default partition spec");
+		if (partitionSpec.fields.length > 0) throw new Error("Storage-engine tables must be unpartitioned.");
+		if (metadata.properties?.["encryption.key-id"] !== void 0) throw new Error("Storage-engine tables must be unencrypted.");
+		const actual = getCurrentSchema(metadata);
+		if (!actual) throw new Error("Table has no current schema");
+		const path = compareFields(expected.fields, actual.fields, "$");
+		if (path) throw new Error(`Table schema does not match at ${path}. Migrate the table or revert the schema change.`);
+		const identifierNames = (schema) => new Set(schema.fields.filter((field) => schema["identifier-field-ids"]?.includes(field.id)).map((field) => field.name));
+		const expectedNames = identifierNames(expected);
+		const actualNames = identifierNames(actual);
+		if (expectedNames.size !== actualNames.size || [...expectedNames].some((name) => !actualNames.has(name))) throw new Error("Table identifier fields do not match. Recreate the table or revert the schema change.");
+		return actual;
+	},
+	catch: (cause) => new ConnectorError({
+		message: cause instanceof Error ? cause.message : "Invalid table metadata",
+		cause
+	})
+});
+/** Creates a missing table through an engine catalog and verifies its schema. */
+const ensureTableFromEngine = (schema, options) => Effect.scoped(Effect.gen(function* () {
+	const catalog = yield* (yield* CatalogManager.make({ engineUri: options.engineUri })).getIcebergCatalog(options.catalogId);
+	return yield* ensureTable(schema, {
+		catalog,
+		identifier: options.identifier,
+		location: options.location,
+		properties: options.properties
+	});
+})).pipe(Effect.mapError(tableError));
+/** Loads a table, creating it when it is missing. */
+const ensureTable = (schema, options) => Effect.gen(function* () {
+	const table = yield* compileTable(schema);
+	const request = (effect) => retryCatalogRequest(effect, options.retry);
+	const metadata = yield* request(options.catalog.loadTable(options.identifier)).pipe(Effect.catchIf((error) => error.status === 404, () => request(options.catalog.createNamespaceIfNotExists({ namespace: options.identifier.namespace })).pipe(Effect.andThen(request(options.catalog.createTable({ namespace: options.identifier.namespace }, makeCreateRequest(table, {
+		name: options.identifier.name,
+		location: options.location,
+		properties: options.properties
+	}))))).pipe(Effect.catchIf((error) => error.status === 409 || error.status === 412, () => Effect.void), Effect.flatMap(() => request(options.catalog.loadTable(options.identifier))))));
+	return yield* verifyTable(metadata, table.schema);
+}).pipe(Effect.mapError(tableError));
+//#endregion
+//#region src/connector-app/create-table.ts
+/** Creates the table command; existing tables must match the connector's row schemas. */
+const makeCreateTableCommand = (schemas) => Command.make("create-table", {
+	table: Argument.String("table"),
+	catalog: Flag.String("catalog"),
+	location: Flag.String("location").pipe(Flag.optional),
+	resource: Flag.String("resource").pipe(Flag.optional)
+}, ({ table, catalog, location, resource }) => Effect.gen(function* () {
+	const namespace = table.split(".");
+	const name = namespace.pop();
+	if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) return yield* new CliError.UserError({
+		cause: "Invalid table name",
+		userMessage: "Use a table name like my.namespace.table"
+	});
+	const resourceName = Option.getOrElse(resource, () => name);
+	const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : void 0;
+	if (!schema) return yield* new CliError.UserError({
+		cause: "Unknown resource",
+		userMessage: `Unknown resource: ${resourceName}`
+	});
+	const uri = yield* engineUri;
+	yield* ensureTableFromEngine(schema, {
+		engineUri: uri.href,
+		catalogId: catalog,
+		identifier: {
+			namespace,
+			name
+		},
+		location: Option.getOrUndefined(location)
+	});
+	yield* Effect.log(`Prepared ${table}`);
+})).pipe(Command.withDescription("Create or verify an Iceberg table for a connector resource"));
+//#endregion
+//#region src/connector-app/demo.ts
+/** Reads byte sizes such as `512kB`, `128MB` or `1.5GB`. */
+const parseSize = (input) => Option.liftThrowable(() => humanFormat.parse.bytes(input))().pipe(Option.filter((bytes) => Number.isSafeInteger(bytes) && bytes > 0));
+/**
+* Creates the command that writes demo data over Arrow Flight, then exits.
+* It reads hosted config and table bindings, so provide `RuntimeConfig.layerHosted()`.
+*/
+const makeDemoCommand = (demo) => Command.make("demo", {
+	size: Flag.String("size").pipe(Flag.filterMap(parseSize, () => "a size like 512kB, 128MB or 1.5GB")),
+	seed: Flag.Int("seed").pipe(Flag.optional),
+	batchSize: Flag.Int("batch-size").pipe(Flag.withSchema(Schema.Int.check(Schema.isGreaterThan(0))), Flag.withDescription("Maximum rows per ingestion batch (default: 500)"), Flag.optional)
+}, ({ size, seed, batchSize }) => Effect.gen(function* () {
+	const now = yield* DateTime.nowAsDate;
+	yield* run(demo, {
+		bytes: size,
+		seed: Option.getOrUndefined(seed),
+		batchSize: Option.getOrUndefined(batchSize),
+		now
+	}).pipe(Effect.provide(layerArrowFlightConfig({ resources: [...demo.setup.tables, ...demo.group.tables] })));
+})).pipe(Command.withDescription("Write demo data for the bound tables, then exit"));
+//#endregion
+//#region src/connector-app/start.ts
+const start = (app, options) => Effect.gen(function* () {
+	const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
+	const routes = app.webhooks?.map((route) => route.path) ?? [];
+	yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
+		port: options.port,
+		routes
+	}));
+	return yield* run$1(app, {
+		initialCutoff,
+		webhook: {
+			routes: app.webhooks ?? [],
+			healthPath: options.healthPath ?? "/health",
+			metricsPath: options.metricsPath,
+			statusPath: options.statusPath,
+			disableHttpLogger: options.disableHttpLogger ?? true
+		}
+	}).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
+});
+//#endregion
+//#region src/connector-app/setup.ts
+/** Creates or verifies all bound tables without loading provider credentials. */
+const makeSetupCommand = (schemas) => Command.make("setup", {
+	checkpointIntervalSec: Flag.Int("checkpoint-interval-sec").pipe(Flag.withSchema(Schema.Int.check(Schema.isBetween({
+		minimum: 0,
+		maximum: 900
+	}))), Flag.withDescription("Checkpoint interval for new tables in seconds (0 disables the interval delay)"), Flag.optional),
+	retryInterval: Flag.String("retry-interval").pipe(Flag.withDefault("5 seconds"), Flag.withSchema(Schema.DurationFromString), Flag.filter((duration) => Duration.isPositive(duration) && Duration.isFinite(duration), () => "a positive duration, such as 5 seconds"), Flag.withDescription("Delay between transient catalog retries")),
+	retries: Flag.Int("retries").pipe(Flag.withDefault(10), Flag.filter((value) => value >= 0, () => "a non-negative number of retries"), Flag.withDescription("Maximum retries per catalog request after the initial attempt"))
+}, ({ retryInterval, retries, checkpointIntervalSec }) => Effect.gen(function* () {
+	const source = { resources: Object.entries(schemas).map(([name, rowSchema]) => ({
+		name,
+		rowSchema
+	})) };
+	const tables = yield* loadTableBindings(source);
+	const uri = yield* engineUri;
+	const catalogId = yield* catalog;
+	const manager = yield* CatalogManager.make({ engineUri: uri.href });
+	const retry = {
+		interval: retryInterval,
+		times: retries
+	};
+	const properties = Option.match(checkpointIntervalSec, {
+		onNone: () => void 0,
+		onSome: (seconds) => ({ "airfoil.checkpoint.interval-sec": String(seconds) })
+	});
+	const catalog$1 = yield* retryCatalogRequest(manager.getIcebergCatalog(catalogId), retry);
+	for (const resource of source.resources) {
+		const binding = tables[resource.name];
+		if (!binding) continue;
+		yield* ensureTable(resource.rowSchema, {
+			catalog: catalog$1,
+			identifier: binding,
+			location: binding.location,
+			retry,
+			properties
+		});
+		yield* Effect.logInfo(`Prepared ${[...binding.namespace, binding.name].join(".")}`);
+	}
+})).pipe(Command.withDescription("Create or verify the Iceberg tables in AIRFOIL_TABLE_BINDINGS"));
+//#endregion
+//#region src/connector-app/index.ts
+var connector_app_exports = /* @__PURE__ */ __exportAll({
+	check: () => check,
+	makeCreateTableCommand: () => makeCreateTableCommand,
+	makeDemoCommand: () => makeDemoCommand,
+	makeSetupCommand: () => makeSetupCommand,
+	parseSize: () => parseSize,
+	start: () => start
+});
+//#endregion
+export { parseSize as a, makeDemoCommand as i, makeSetupCommand as n, makeCreateTableCommand as o, start as r, check as s, connector_app_exports as t };
+
+//# sourceMappingURL=connector-app-C6_dZnhE.mjs.map

package/dist/connector-app-C6_dZnhE.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"connector-app-C6_dZnhE.mjs","names":["runIngestion","RuntimeConfig.loadTableBindings","RuntimeConfig.engineUri","RuntimeConfig.catalog","catalog"],"sources":["../src/connector-app/check.ts","../src/catalog/retry.ts","../src/catalog/table.ts","../src/connector-app/create-table.ts","../src/connector-app/demo.ts","../src/connector-app/start.ts","../src/connector-app/setup.ts","../src/connector-app/index.ts"],"sourcesContent":["import { Context, Effect, Layer, Tuple } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition, ResourceName } from \"../core/types\";\n\nexport type ResourceCheckResult =\n  | { readonly _tag: \"ok\" }\n  | { readonly _tag: \"error\"; readonly message: string };\n\nexport type CheckOptions<Name extends string> = {\n  readonly resources: ReadonlyArray<Name>;\n};\n\ntype ResourceCheckResults<Name extends PropertyKey> = {\n  readonly [Key in Name]: ResourceCheckResult;\n};\n\nexport type CheckResult<\n  Resources extends ReadonlyArray<ResourceDefinition>,\n  Name extends ResourceName<Resources> = ResourceName<Resources>,\n> = ResourceCheckResults<Name>;\n\ntype CheckFailure = { readonly message: string };\n\nconst ok: ResourceCheckResult = { _tag: \"ok\" };\n\nconst error = (message: string): ResourceCheckResult => ({ _tag: \"error\", message });\n\nconst fromEntries = <Name extends string>(\n  entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,\n): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;\n\nconst resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>\n  fromEntries(names.map((name) => Tuple.make(name, result)));\n\n/**\n * Builds the connector service from its layer and validates only the selected\n * resources. The layer remains open until every check finishes.\n */\nexport function check<\n  Identifier,\n  const Resources extends ReadonlyArray<ResourceDefinition>,\n  E extends CheckFailure,\n  R,\n  const Name extends ResourceName<Resources>,\n>(\n  service: Context.Key<Identifier, ConnectorDefinition<Resources>>,\n  layer: Layer.Layer<Identifier, E, R>,\n  options: CheckOptions<Name>,\n): Effect.Effect<CheckResult<Resources, Name>, never, R> {\n  if (options.resources.length === 0) {\n    return Effect.succeed(fromEntries<Name>([]));\n  }\n\n  return service.pipe(\n    Effect.flatMap((connector) =>\n      Effect.forEach(options.resources, (name) => {\n        const resource = connector.resources.find((candidate) => candidate.name === name);\n        const result = resource\n          ? resource.check.pipe(\n              Effect.match({\n                onFailure: (failure) => error(failure.message),\n                onSuccess: () => ok,\n              }),\n            )\n          : Effect.succeed(error(`Unknown connector resource: ${name}`));\n\n        return result.pipe(Effect.map((result) => Tuple.make(name, result)));\n      }).pipe(Effect.map(fromEntries)),\n    ),\n    Effect.provide(layer),\n    Effect.match({\n      onFailure: (failure) => resultsFor(options.resources, error(failure.message)),\n      onSuccess: (results) => results,\n    }),\n  );\n}\n","import { Effect, Schedule, type Duration } from \"effect\";\n\nexport type TableRetryOptions = {\n  readonly interval: Duration.Input;\n  /** Retries after the initial attempt. */\n  readonly times: number;\n};\n\n/** Retries transient catalog responses at the request boundary. */\nexport const retryCatalogRequest = <A, E extends { readonly status?: number }, R>(\n  request: Effect.Effect<A, E, R>,\n  options?: TableRetryOptions,\n) =>\n  options === undefined\n    ? request\n    : request.pipe(\n        Effect.retry({\n          schedule: Schedule.spaced(options.interval),\n          times: options.times,\n          while: (error) =>\n            error.status === 408 ||\n            error.status === 429 ||\n            (error.status !== undefined && error.status >= 500 && error.status <= 599),\n        }),\n      );\n","import type {\n  IcebergType,\n  StructField,\n  TableIdentifier,\n  TableMetadata,\n  TableSchema,\n} from \"iceberg-js\";\n\nimport { CatalogManager } from \"@useairfoil/api\";\nimport { Effect, type Schema } from \"effect\";\nimport { HttpClient } from \"effect/http\";\nimport { getCurrentSchema } from \"iceberg-js\";\n\nimport { ConnectorError } from \"../errors\";\nimport { IcebergSchemaError } from \"../iceberg/error\";\nimport { compileTable, makeCreateRequest } from \"../iceberg/table\";\nimport { retryCatalogRequest, type TableRetryOptions } from \"./retry\";\n\ntype Catalog = Effect.Success<\n  ReturnType<CatalogManager.CatalogManagerService[\"getIcebergCatalog\"]>\n>;\n\ntype TableOptions = {\n  readonly retry?: TableRetryOptions;\n  readonly catalog: Pick<Catalog, \"loadTable\" | \"createTable\" | \"createNamespaceIfNotExists\">;\n  readonly identifier: TableIdentifier;\n  readonly location?: string | undefined;\n  readonly properties?: Readonly<Record<string, string>> | undefined;\n};\n\ntype EngineTableOptions = {\n  readonly engineUri: string;\n  readonly catalogId: string;\n  readonly identifier: TableIdentifier;\n  readonly location?: string | undefined;\n  readonly properties?: Readonly<Record<string, string>> | undefined;\n};\n\nconst tableError = (cause: unknown): ConnectorError => {\n  if (cause instanceof ConnectorError) return cause;\n\n  const message = cause instanceof Error ? cause.message : \"Failed to prepare Iceberg table\";\n  return new ConnectorError({\n    message: cause instanceof IcebergSchemaError ? `${cause.path}: ${message}` : message,\n    cause,\n  });\n};\n\nconst compareTypes = (\n  expected: IcebergType,\n  actual: IcebergType,\n  path: string,\n): string | undefined => {\n  if (typeof expected === \"string\" || typeof actual === \"string\") {\n    return expected === actual ? undefined : path;\n  }\n\n  if (expected.type !== actual.type) return path;\n\n  switch (expected.type) {\n    case \"struct\":\n      if (actual.type !== \"struct\") return path;\n      return compareFields(expected.fields, actual.fields, path);\n    case \"list\":\n      if (actual.type !== \"list\" || expected[\"element-required\"] !== actual[\"element-required\"]) {\n        return path;\n      }\n      return compareTypes(expected.element, actual.element, `${path}.element`);\n    case \"map\":\n      if (actual.type !== \"map\" || expected[\"value-required\"] !== actual[\"value-required\"]) {\n        return path;\n      }\n      return (\n        compareTypes(expected.key, actual.key, `${path}.key`) ??\n        compareTypes(expected.value, actual.value, `${path}.value`)\n      );\n  }\n};\n\nconst compareFields = (\n  expected: ReadonlyArray<StructField>,\n  actual: ReadonlyArray<StructField>,\n  path: string,\n): string | undefined => {\n  if (expected.length !== actual.length) return path;\n\n  for (const field of expected) {\n    const fieldPath = `${path}.${field.name}`;\n    const other = actual.find((candidate) => candidate.name === field.name);\n\n    if (!other || field.required !== other.required) return fieldPath;\n\n    const mismatch = compareTypes(field.type, other.type, fieldPath);\n    if (mismatch) return mismatch;\n  }\n};\n\nconst verifyTable = (\n  metadata: TableMetadata,\n  expected: TableSchema,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      if (!Array.isArray(metadata.schemas)) throw new Error(\"Invalid table metadata\");\n      if (metadata[\"format-version\"] !== 3) {\n        throw new Error(\n          \"Storage-engine tables must use Iceberg format v3. Migrate the table before ingesting.\",\n        );\n      }\n      const partitionSpec = metadata[\"partition-specs\"].find(\n        (spec) => spec[\"spec-id\"] === metadata[\"default-spec-id\"],\n      );\n      if (!partitionSpec) throw new Error(\"Table has no default partition spec\");\n      if (partitionSpec.fields.length > 0) {\n        throw new Error(\"Storage-engine tables must be unpartitioned.\");\n      }\n      if (metadata.properties?.[\"encryption.key-id\"] !== undefined) {\n        throw new Error(\"Storage-engine tables must be unencrypted.\");\n      }\n      const actual = getCurrentSchema(metadata);\n      if (!actual) throw new Error(\"Table has no current schema\");\n\n      const path = compareFields(expected.fields, actual.fields, \"$\");\n      if (path) {\n        throw new Error(\n          `Table schema does not match at ${path}. Migrate the table or revert the schema change.`,\n        );\n      }\n\n      const identifierNames = (schema: TableSchema) =>\n        new Set(\n          schema.fields\n            .filter((field) => schema[\"identifier-field-ids\"]?.includes(field.id))\n            .map((field) => field.name),\n        );\n      const expectedNames = identifierNames(expected);\n      const actualNames = identifierNames(actual);\n      if (\n        expectedNames.size !== actualNames.size ||\n        [...expectedNames].some((name) => !actualNames.has(name))\n      ) {\n        throw new Error(\n          \"Table identifier fields do not match. Recreate the table or revert the schema change.\",\n        );\n      }\n\n      return actual;\n    },\n    catch: (cause) =>\n      new ConnectorError({\n        message: cause instanceof Error ? cause.message : \"Invalid table metadata\",\n        cause,\n      }),\n  });\n\n/** Creates a table through the REST catalog, then checks the loaded schema. */\nexport const createTable = (\n  schema: Schema.Top,\n  options: TableOptions,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.gen(function* () {\n    const table = yield* compileTable(schema);\n    yield* options.catalog.createTable(\n      { namespace: options.identifier.namespace },\n      makeCreateRequest(table, {\n        name: options.identifier.name,\n        location: options.location,\n        properties: options.properties,\n      }),\n    );\n\n    const metadata = yield* options.catalog.loadTable(options.identifier);\n    return yield* verifyTable(metadata, table.schema);\n  }).pipe(Effect.mapError(tableError));\n\n/** Creates a missing table through an engine catalog and verifies its schema. */\nexport const ensureTableFromEngine = (\n  schema: Schema.Top,\n  options: EngineTableOptions,\n): Effect.Effect<TableSchema, ConnectorError, HttpClient.HttpClient> =>\n  Effect.scoped(\n    Effect.gen(function* () {\n      const manager = yield* CatalogManager.make({ engineUri: options.engineUri });\n      const catalog = yield* manager.getIcebergCatalog(options.catalogId);\n      return yield* ensureTable(schema, {\n        catalog,\n        identifier: options.identifier,\n        location: options.location,\n        properties: options.properties,\n      });\n    }),\n  ).pipe(Effect.mapError(tableError));\n\n/** Loads a table, creating it when it is missing. */\nexport const ensureTable = (\n  schema: Schema.Top,\n  options: TableOptions,\n): Effect.Effect<TableSchema, ConnectorError> =>\n  Effect.gen(function* () {\n    const table = yield* compileTable(schema);\n    const request = <A>(\n      effect: Effect.Effect<A, Effect.Error<ReturnType<Catalog[\"loadConfig\"]>>>,\n    ) => retryCatalogRequest(effect, options.retry);\n    const metadata = yield* request(options.catalog.loadTable(options.identifier)).pipe(\n      Effect.catchIf(\n        (error) => error.status === 404,\n        () =>\n          request(\n            options.catalog.createNamespaceIfNotExists({ namespace: options.identifier.namespace }),\n          )\n            .pipe(\n              Effect.andThen(\n                request(\n                  options.catalog.createTable(\n                    { namespace: options.identifier.namespace },\n                    makeCreateRequest(table, {\n                      name: options.identifier.name,\n                      location: options.location,\n                      properties: options.properties,\n                    }),\n                  ),\n                ),\n              ),\n            )\n            .pipe(\n              // Another instance may have created the table. Reload and check it.\n              Effect.catchIf(\n                (error) => error.status === 409 || error.status === 412,\n                () => Effect.void,\n              ),\n              Effect.flatMap(() => request(options.catalog.loadTable(options.identifier))),\n            ),\n      ),\n    );\n\n    return yield* verifyTable(metadata, table.schema);\n  }).pipe(Effect.mapError(tableError));\n","import { Effect, Option, type Schema } from \"effect\";\nimport { Argument, CliError, Command, Flag } from \"effect/cli\";\n\nimport { ensureTableFromEngine } from \"../catalog/table\";\nimport { engineUri } from \"../runtime-config/config\";\n\n/** Creates the table command; existing tables must match the connector's row schemas. */\nexport const makeCreateTableCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>\n  Command.make(\n    \"create-table\",\n    {\n      table: Argument.String(\"table\"),\n      catalog: Flag.String(\"catalog\"),\n      location: Flag.String(\"location\").pipe(Flag.optional),\n      resource: Flag.String(\"resource\").pipe(Flag.optional),\n    },\n    ({ table, catalog, location, resource }) =>\n      Effect.gen(function* () {\n        const namespace = table.split(\".\");\n        const name = namespace.pop();\n\n        if (!name || namespace.length === 0 || namespace.some((part) => part.length === 0)) {\n          return yield* new CliError.UserError({\n            cause: \"Invalid table name\",\n            userMessage: \"Use a table name like my.namespace.table\",\n          });\n        }\n\n        const resourceName = Option.getOrElse(resource, () => name);\n        const schema = Object.hasOwn(schemas, resourceName) ? schemas[resourceName] : undefined;\n        if (!schema) {\n          return yield* new CliError.UserError({\n            cause: \"Unknown resource\",\n            userMessage: `Unknown resource: ${resourceName}`,\n          });\n        }\n\n        const uri = yield* engineUri;\n        yield* ensureTableFromEngine(schema, {\n          engineUri: uri.href,\n          catalogId: catalog,\n          identifier: { namespace, name },\n          location: Option.getOrUndefined(location),\n        });\n        yield* Effect.log(`Prepared ${table}`);\n      }),\n  ).pipe(Command.withDescription(\"Create or verify an Iceberg table for a connector resource\"));\n","import { DateTime, Effect, Option, Schema } from \"effect\";\nimport { Command, Flag } from \"effect/cli\";\nimport humanFormat from \"human-format\";\n\nimport { run, type DemoDefinition } from \"../demo\";\nimport { layerArrowFlightConfig } from \"../ingestor/arrow-flight\";\n\n/** Reads byte sizes such as `512kB`, `128MB` or `1.5GB`. */\nexport const parseSize = (input: string): Option.Option<number> =>\n  Option.liftThrowable(() => humanFormat.parse.bytes(input))().pipe(\n    Option.filter((bytes) => Number.isSafeInteger(bytes) && bytes > 0),\n  );\n\n/**\n * Creates the command that writes demo data over Arrow Flight, then exits.\n * It reads hosted config and table bindings, so provide `RuntimeConfig.layerHosted()`.\n */\nexport const makeDemoCommand = (demo: DemoDefinition) =>\n  Command.make(\n    \"demo\",\n    {\n      size: Flag.String(\"size\").pipe(\n        Flag.filterMap(parseSize, () => \"a size like 512kB, 128MB or 1.5GB\"),\n      ),\n      seed: Flag.Int(\"seed\").pipe(Flag.optional),\n      batchSize: Flag.Int(\"batch-size\").pipe(\n        Flag.withSchema(Schema.Int.check(Schema.isGreaterThan(0))),\n        Flag.withDescription(\"Maximum rows per ingestion batch (default: 500)\"),\n        Flag.optional,\n      ),\n    },\n    ({ size, seed, batchSize }) =>\n      Effect.gen(function* () {\n        const now = yield* DateTime.nowAsDate;\n\n        yield* run(demo, {\n          bytes: size,\n          seed: Option.getOrUndefined(seed),\n          batchSize: Option.getOrUndefined(batchSize),\n          now,\n        }).pipe(\n          Effect.provide(\n            layerArrowFlightConfig({ resources: [...demo.setup.tables, ...demo.group.tables] }),\n          ),\n        );\n      }),\n  ).pipe(Command.withDescription(\"Write demo data for the bound tables, then exit\"));\n","import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n  readonly port: number;\n  readonly initialCutoff?: Cursor.Value;\n  readonly healthPath?: HttpRouter.PathInput;\n  readonly metricsPath?: HttpRouter.PathInput | false;\n  readonly statusPath?: HttpRouter.PathInput | false;\n  readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n  Effect.gen(function* () {\n    const initialCutoff =\n      options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n    const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n    yield* Effect.logInfo(\"webhook server ready\").pipe(\n      Effect.annotateLogs({ port: options.port, routes }),\n    );\n\n    return yield* runIngestion(app, {\n      initialCutoff,\n      webhook: {\n        routes: app.webhooks ?? [],\n        healthPath: options.healthPath ?? \"/health\",\n        metricsPath: options.metricsPath,\n        statusPath: options.statusPath,\n        disableHttpLogger: options.disableHttpLogger ?? true,\n      },\n    }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n  });\n","import { CatalogManager } from \"@useairfoil/api\";\nimport { Duration, Effect, Option, Schema } from \"effect\";\nimport { Command, Flag } from \"effect/cli\";\n\nimport { retryCatalogRequest } from \"../catalog/retry\";\nimport { ensureTable } from \"../catalog/table\";\nimport * as RuntimeConfig from \"../runtime-config\";\n\n/** Creates or verifies all bound tables without loading provider credentials. */\nexport const makeSetupCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>\n  Command.make(\n    \"setup\",\n    {\n      checkpointIntervalSec: Flag.Int(\"checkpoint-interval-sec\").pipe(\n        Flag.withSchema(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 900 }))),\n        Flag.withDescription(\n          \"Checkpoint interval for new tables in seconds (0 disables the interval delay)\",\n        ),\n        Flag.optional,\n      ),\n      retryInterval: Flag.String(\"retry-interval\").pipe(\n        Flag.withDefault(\"5 seconds\"),\n        Flag.withSchema(Schema.DurationFromString),\n        Flag.filter(\n          (duration) => Duration.isPositive(duration) && Duration.isFinite(duration),\n          () => \"a positive duration, such as 5 seconds\",\n        ),\n        Flag.withDescription(\"Delay between transient catalog retries\"),\n      ),\n      retries: Flag.Int(\"retries\").pipe(\n        Flag.withDefault(10),\n        Flag.filter(\n          (value) => value >= 0,\n          () => \"a non-negative number of retries\",\n        ),\n        Flag.withDescription(\"Maximum retries per catalog request after the initial attempt\"),\n      ),\n    },\n    ({ retryInterval, retries, checkpointIntervalSec }) =>\n      Effect.gen(function* () {\n        const source = {\n          resources: Object.entries(schemas).map(([name, rowSchema]) => ({ name, rowSchema })),\n        };\n        const tables = yield* RuntimeConfig.loadTableBindings(source);\n        const uri = yield* RuntimeConfig.engineUri;\n        const catalogId = yield* RuntimeConfig.catalog;\n        const manager = yield* CatalogManager.make({ engineUri: uri.href });\n        const retry = { interval: retryInterval, times: retries };\n        const properties = Option.match(checkpointIntervalSec, {\n          onNone: () => undefined,\n          onSome: (seconds) => ({ \"airfoil.checkpoint.interval-sec\": String(seconds) }),\n        });\n        const catalog = yield* retryCatalogRequest(manager.getIcebergCatalog(catalogId), retry);\n\n        // Sequential setup avoids bursting the catalog's table-creation quota.\n        for (const resource of source.resources) {\n          const binding = tables[resource.name];\n          if (!binding) continue;\n          yield* ensureTable(resource.rowSchema, {\n            catalog,\n            identifier: binding,\n            location: binding.location,\n            retry,\n            properties,\n          });\n          yield* Effect.logInfo(`Prepared ${[...binding.namespace, binding.name].join(\".\")}`);\n        }\n      }),\n  ).pipe(Command.withDescription(\"Create or verify the Iceberg tables in AIRFOIL_TABLE_BINDINGS\"));\n","export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from \"./check\";\nexport { makeCreateTableCommand } from \"./create-table\";\nexport { makeDemoCommand, parseSize } from \"./demo\";\nexport { start, type App, type StartOptions } from \"./start\";\nexport { makeSetupCommand } from \"./setup\";\n"],"mappings":";;;;;;;;;;;;;;;;AAuBA,MAAM,KAA0B,EAAE,MAAM,KAAK;AAE7C,MAAM,SAAS,aAA0C;CAAE,MAAM;CAAS;AAAQ;AAElF,MAAM,eACJ,YAC+B,OAAO,YAAY,OAAO;AAE3D,MAAM,cAAmC,OAA4B,WACnE,YAAY,MAAM,KAAK,SAAS,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;;;;;AAM3D,SAAgB,MAOd,SACA,OACA,SACuD;CACvD,IAAI,QAAQ,UAAU,WAAW,GAC/B,OAAO,OAAO,QAAQ,YAAkB,CAAC,CAAC,CAAC;CAG7C,OAAO,QAAQ,KACb,OAAO,SAAS,cACd,OAAO,QAAQ,QAAQ,YAAY,SAAS;EAC1C,MAAM,WAAW,UAAU,UAAU,MAAM,cAAc,UAAU,SAAS,IAAI;EAUhF,QATe,WACX,SAAS,MAAM,KACb,OAAO,MAAM;GACX,YAAY,YAAY,MAAM,QAAQ,OAAO;GAC7C,iBAAiB;EACnB,CAAC,CACH,IACA,OAAO,QAAQ,MAAM,+BAA+B,MAAM,CAAC,EAAA,CAEjD,KAAK,OAAO,KAAK,WAAW,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;CACrE,CAAC,CAAC,CAAC,KAAK,OAAO,IAAI,WAAW,CAAC,CACjC,GACA,OAAO,QAAQ,KAAK,GACpB,OAAO,MAAM;EACX,YAAY,YAAY,WAAW,QAAQ,WAAW,MAAM,QAAQ,OAAO,CAAC;EAC5E,YAAY,YAAY;CAC1B,CAAC,CACH;AACF;;;;AClEA,MAAa,uBACX,SACA,YAEA,YAAY,KAAA,IACR,UACA,QAAQ,KACN,OAAO,MAAM;CACX,UAAU,SAAS,OAAO,QAAQ,QAAQ;CAC1C,OAAO,QAAQ;CACf,QAAQ,UACN,MAAM,WAAW,OACjB,MAAM,WAAW,OAChB,MAAM,WAAW,KAAA,KAAa,MAAM,UAAU,OAAO,MAAM,UAAU;AAC1E,CAAC,CACH;;;ACcN,MAAM,cAAc,UAAmC;CACrD,IAAI,iBAAiB,gBAAgB,OAAO;CAE5C,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;CACzD,OAAO,IAAI,eAAe;EACxB,SAAS,iBAAiB,qBAAqB,GAAG,MAAM,KAAK,IAAI,YAAY;EAC7E;CACF,CAAC;AACH;AAEA,MAAM,gBACJ,UACA,QACA,SACuB;CACvB,IAAI,OAAO,aAAa,YAAY,OAAO,WAAW,UACpD,OAAO,aAAa,SAAS,KAAA,IAAY;CAG3C,IAAI,SAAS,SAAS,OAAO,MAAM,OAAO;CAE1C,QAAQ,SAAS,MAAjB;EACE,KAAK;GACH,IAAI,OAAO,SAAS,UAAU,OAAO;GACrC,OAAO,cAAc,SAAS,QAAQ,OAAO,QAAQ,IAAI;EAC3D,KAAK;GACH,IAAI,OAAO,SAAS,UAAU,SAAS,wBAAwB,OAAO,qBACpE,OAAO;GAET,OAAO,aAAa,SAAS,SAAS,OAAO,SAAS,GAAG,KAAK,SAAS;EACzE,KAAK;GACH,IAAI,OAAO,SAAS,SAAS,SAAS,sBAAsB,OAAO,mBACjE,OAAO;GAET,OACE,aAAa,SAAS,KAAK,OAAO,KAAK,GAAG,KAAK,KAAK,KACpD,aAAa,SAAS,OAAO,OAAO,OAAO,GAAG,KAAK,OAAO;CAEhE;AACF;AAEA,MAAM,iBACJ,UACA,QACA,SACuB;CACvB,IAAI,SAAS,WAAW,OAAO,QAAQ,OAAO;CAE9C,KAAK,MAAM,SAAS,UAAU;EAC5B,MAAM,YAAY,GAAG,KAAK,GAAG,MAAM;EACnC,MAAM,QAAQ,OAAO,MAAM,cAAc,UAAU,SAAS,MAAM,IAAI;EAEtE,IAAI,CAAC,SAAS,MAAM,aAAa,MAAM,UAAU,OAAO;EAExD,MAAM,WAAW,aAAa,MAAM,MAAM,MAAM,MAAM,SAAS;EAC/D,IAAI,UAAU,OAAO;CACvB;AACF;AAEA,MAAM,eACJ,UACA,aAEA,OAAO,IAAI;CACT,WAAW;EACT,IAAI,CAAC,MAAM,QAAQ,SAAS,OAAO,GAAG,MAAM,IAAI,MAAM,wBAAwB;EAC9E,IAAI,SAAS,sBAAsB,GACjC,MAAM,IAAI,MACR,uFACF;EAEF,MAAM,gBAAgB,SAAS,kBAAkB,CAAC,MAC/C,SAAS,KAAK,eAAe,SAAS,kBACzC;EACA,IAAI,CAAC,eAAe,MAAM,IAAI,MAAM,qCAAqC;EACzE,IAAI,cAAc,OAAO,SAAS,GAChC,MAAM,IAAI,MAAM,8CAA8C;EAEhE,IAAI,SAAS,aAAa,yBAAyB,KAAA,GACjD,MAAM,IAAI,MAAM,4CAA4C;EAE9D,MAAM,SAAS,iBAAiB,QAAQ;EACxC,IAAI,CAAC,QAAQ,MAAM,IAAI,MAAM,6BAA6B;EAE1D,MAAM,OAAO,cAAc,SAAS,QAAQ,OAAO,QAAQ,GAAG;EAC9D,IAAI,MACF,MAAM,IAAI,MACR,kCAAkC,KAAK,iDACzC;EAGF,MAAM,mBAAmB,WACvB,IAAI,IACF,OAAO,OACJ,QAAQ,UAAU,OAAO,uBAAuB,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CACrE,KAAK,UAAU,MAAM,IAAI,CAC9B;EACF,MAAM,gBAAgB,gBAAgB,QAAQ;EAC9C,MAAM,cAAc,gBAAgB,MAAM;EAC1C,IACE,cAAc,SAAS,YAAY,QACnC,CAAC,GAAG,aAAa,CAAC,CAAC,MAAM,SAAS,CAAC,YAAY,IAAI,IAAI,CAAC,GAExD,MAAM,IAAI,MACR,uFACF;EAGF,OAAO;CACT;CACA,QAAQ,UACN,IAAI,eAAe;EACjB,SAAS,iBAAiB,QAAQ,MAAM,UAAU;EAClD;CACF,CAAC;AACL,CAAC;;AAuBH,MAAa,yBACX,QACA,YAEA,OAAO,OACL,OAAO,IAAI,aAAa;CAEtB,MAAM,UAAU,QAAO,OADA,eAAe,KAAK,EAAE,WAAW,QAAQ,UAAU,CAAC,EAAA,CAC5C,kBAAkB,QAAQ,SAAS;CAClE,OAAO,OAAO,YAAY,QAAQ;EAChC;EACA,YAAY,QAAQ;EACpB,UAAU,QAAQ;EAClB,YAAY,QAAQ;CACtB,CAAC;AACH,CAAC,CACH,CAAC,CAAC,KAAK,OAAO,SAAS,UAAU,CAAC;;AAGpC,MAAa,eACX,QACA,YAEA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,aAAa,MAAM;CACxC,MAAM,WACJ,WACG,oBAAoB,QAAQ,QAAQ,KAAK;CAC9C,MAAM,WAAW,OAAO,QAAQ,QAAQ,QAAQ,UAAU,QAAQ,UAAU,CAAC,CAAC,CAAC,KAC7E,OAAO,SACJ,UAAU,MAAM,WAAW,WAE1B,QACE,QAAQ,QAAQ,2BAA2B,EAAE,WAAW,QAAQ,WAAW,UAAU,CAAC,CACxF,CAAC,CACE,KACC,OAAO,QACL,QACE,QAAQ,QAAQ,YACd,EAAE,WAAW,QAAQ,WAAW,UAAU,GAC1C,kBAAkB,OAAO;EACvB,MAAM,QAAQ,WAAW;EACzB,UAAU,QAAQ;EAClB,YAAY,QAAQ;CACtB,CAAC,CACH,CACF,CACF,CACF,CAAC,CACA,KAEC,OAAO,SACJ,UAAU,MAAM,WAAW,OAAO,MAAM,WAAW,WAC9C,OAAO,IACf,GACA,OAAO,cAAc,QAAQ,QAAQ,QAAQ,UAAU,QAAQ,UAAU,CAAC,CAAC,CAC7E,CACN,CACF;CAEA,OAAO,OAAO,YAAY,UAAU,MAAM,MAAM;AAClD,CAAC,CAAC,CAAC,KAAK,OAAO,SAAS,UAAU,CAAC;;;;ACrOrC,MAAa,0BAA0B,YACrC,QAAQ,KACN,gBACA;CACE,OAAO,SAAS,OAAO,OAAO;CAC9B,SAAS,KAAK,OAAO,SAAS;CAC9B,UAAU,KAAK,OAAO,UAAU,CAAC,CAAC,KAAK,KAAK,QAAQ;CACpD,UAAU,KAAK,OAAO,UAAU,CAAC,CAAC,KAAK,KAAK,QAAQ;AACtD,IACC,EAAE,OAAO,SAAS,UAAU,eAC3B,OAAO,IAAI,aAAa;CACtB,MAAM,YAAY,MAAM,MAAM,GAAG;CACjC,MAAM,OAAO,UAAU,IAAI;CAE3B,IAAI,CAAC,QAAQ,UAAU,WAAW,KAAK,UAAU,MAAM,SAAS,KAAK,WAAW,CAAC,GAC/E,OAAO,OAAO,IAAI,SAAS,UAAU;EACnC,OAAO;EACP,aAAa;CACf,CAAC;CAGH,MAAM,eAAe,OAAO,UAAU,gBAAgB,IAAI;CAC1D,MAAM,SAAS,OAAO,OAAO,SAAS,YAAY,IAAI,QAAQ,gBAAgB,KAAA;CAC9E,IAAI,CAAC,QACH,OAAO,OAAO,IAAI,SAAS,UAAU;EACnC,OAAO;EACP,aAAa,qBAAqB;CACpC,CAAC;CAGH,MAAM,MAAM,OAAO;CACnB,OAAO,sBAAsB,QAAQ;EACnC,WAAW,IAAI;EACf,WAAW;EACX,YAAY;GAAE;GAAW;EAAK;EAC9B,UAAU,OAAO,eAAe,QAAQ;CAC1C,CAAC;CACD,OAAO,OAAO,IAAI,YAAY,OAAO;AACvC,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,4DAA4D,CAAC;;;;ACtC9F,MAAa,aAAa,UACxB,OAAO,oBAAoB,YAAY,MAAM,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAC3D,OAAO,QAAQ,UAAU,OAAO,cAAc,KAAK,KAAK,QAAQ,CAAC,CACnE;;;;;AAMF,MAAa,mBAAmB,SAC9B,QAAQ,KACN,QACA;CACE,MAAM,KAAK,OAAO,MAAM,CAAC,CAAC,KACxB,KAAK,UAAU,iBAAiB,mCAAmC,CACrE;CACA,MAAM,KAAK,IAAI,MAAM,CAAC,CAAC,KAAK,KAAK,QAAQ;CACzC,WAAW,KAAK,IAAI,YAAY,CAAC,CAAC,KAChC,KAAK,WAAW,OAAO,IAAI,MAAM,OAAO,cAAc,CAAC,CAAC,CAAC,GACzD,KAAK,gBAAgB,iDAAiD,GACtE,KAAK,QACP;AACF,IACC,EAAE,MAAM,MAAM,gBACb,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,SAAS;CAE5B,OAAO,IAAI,MAAM;EACf,OAAO;EACP,MAAM,OAAO,eAAe,IAAI;EAChC,WAAW,OAAO,eAAe,SAAS;EAC1C;CACF,CAAC,CAAC,CAAC,KACD,OAAO,QACL,uBAAuB,EAAE,WAAW,CAAC,GAAG,KAAK,MAAM,QAAQ,GAAG,KAAK,MAAM,MAAM,EAAE,CAAC,CACpF,CACF;AACF,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,iDAAiD,CAAC;;;ACzBnF,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,SAAS,CAAC;CACnF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,IAAI,KAAK,CAAC;CAE5D,OAAO,OAAO,QAAQ,sBAAsB,CAAC,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;CAAO,CAAC,CACpD;CAEA,OAAO,OAAOA,MAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,CAAC;GACzB,YAAY,QAAQ,cAAc;GAClC,aAAa,QAAQ;GACrB,YAAY,QAAQ;GACpB,mBAAmB,QAAQ,qBAAqB;EAClD;CACF,CAAC,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,KAAK,CAAC,CAAC,CAAC;AACpF,CAAC;;;;AChCH,MAAa,oBAAoB,YAC/B,QAAQ,KACN,SACA;CACE,uBAAuB,KAAK,IAAI,yBAAyB,CAAC,CAAC,KACzD,KAAK,WAAW,OAAO,IAAI,MAAM,OAAO,UAAU;EAAE,SAAS;EAAG,SAAS;CAAI,CAAC,CAAC,CAAC,GAChF,KAAK,gBACH,+EACF,GACA,KAAK,QACP;CACA,eAAe,KAAK,OAAO,gBAAgB,CAAC,CAAC,KAC3C,KAAK,YAAY,WAAW,GAC5B,KAAK,WAAW,OAAO,kBAAkB,GACzC,KAAK,QACF,aAAa,SAAS,WAAW,QAAQ,KAAK,SAAS,SAAS,QAAQ,SACnE,wCACR,GACA,KAAK,gBAAgB,yCAAyC,CAChE;CACA,SAAS,KAAK,IAAI,SAAS,CAAC,CAAC,KAC3B,KAAK,YAAY,EAAE,GACnB,KAAK,QACF,UAAU,SAAS,SACd,kCACR,GACA,KAAK,gBAAgB,+DAA+D,CACtF;AACF,IACC,EAAE,eAAe,SAAS,4BACzB,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,EACb,WAAW,OAAO,QAAQ,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,gBAAgB;EAAE;EAAM;CAAU,EAAE,EACrF;CACA,MAAM,SAAS,OAAOC,kBAAgC,MAAM;CAC5D,MAAM,MAAM,OAAOC;CACnB,MAAM,YAAY,OAAOC;CACzB,MAAM,UAAU,OAAO,eAAe,KAAK,EAAE,WAAW,IAAI,KAAK,CAAC;CAClE,MAAM,QAAQ;EAAE,UAAU;EAAe,OAAO;CAAQ;CACxD,MAAM,aAAa,OAAO,MAAM,uBAAuB;EACrD,cAAc,KAAA;EACd,SAAS,aAAa,EAAE,mCAAmC,OAAO,OAAO,EAAE;CAC7E,CAAC;CACD,MAAMC,YAAU,OAAO,oBAAoB,QAAQ,kBAAkB,SAAS,GAAG,KAAK;CAGtF,KAAK,MAAM,YAAY,OAAO,WAAW;EACvC,MAAM,UAAU,OAAO,SAAS;EAChC,IAAI,CAAC,SAAS;EACd,OAAO,YAAY,SAAS,WAAW;GACrC,SAAA;GACA,YAAY;GACZ,UAAU,QAAQ;GAClB;GACA;EACF,CAAC;EACD,OAAO,OAAO,QAAQ,YAAY,CAAC,GAAG,QAAQ,WAAW,QAAQ,IAAI,CAAC,CAAC,KAAK,GAAG,GAAG;CACpF;AACF,CAAC,CACL,CAAC,CAAC,KAAK,QAAQ,gBAAgB,+DAA+D,CAAC"}

package/dist/{constants-D-GQzpHZ.mjs → constants-v3UX-hf6.mjs}

@@ -8,7 +8,8 @@ const PlatformRuntimeKey = {
 	connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
 	stateTable: "AIRFOIL_STATE_TABLE",
 	postgresConnectionString: "POSTGRES_CONNECTION_STRING",
-	wingsUri: "WINGS_URI",
+	engineUri: "ENGINE_URI",
+	flightUri: "FLIGHT_URI",
 	catalog: "AIRFOIL_CATALOG",
 	otelEnabled: "OTEL_ENABLED",
 	otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
@@ -28,4 +29,4 @@ const isPlatformRuntimeKey = (key) => HashSet.has(platformRuntimeKeys, key);
 //#endregion
 export { PlatformRuntimeKey as n, isPlatformRuntimeKey as r, PlatformRuntimeDefault as t };
 
-//# sourceMappingURL=constants-D-GQzpHZ.mjs.map
+//# sourceMappingURL=constants-v3UX-hf6.mjs.map

package/dist/constants-v3UX-hf6.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants-v3UX-hf6.mjs","names":[],"sources":["../src/runtime-config/constants.ts"],"sourcesContent":["import { HashSet } from \"effect\";\n\n/** Shared runtime keys owned by Airfoil rather than an individual connector. */\nexport const PlatformRuntimeKey = {\n  configPath: \"AIRFOIL_CONFIG_PATH\",\n  httpPort: \"AIRFOIL_HTTP_PORT\",\n  tableBindings: \"AIRFOIL_TABLE_BINDINGS\",\n  connectorInstanceId: \"AIRFOIL_CONNECTOR_INSTANCE_ID\",\n  stateTable: \"AIRFOIL_STATE_TABLE\",\n  postgresConnectionString: \"POSTGRES_CONNECTION_STRING\",\n  engineUri: \"ENGINE_URI\",\n  flightUri: \"FLIGHT_URI\",\n  catalog: \"AIRFOIL_CATALOG\",\n  otelEnabled: \"OTEL_ENABLED\",\n  otelExporterOtlpEndpoint: \"OTEL_EXPORTER_OTLP_ENDPOINT\",\n  otelExporterOtlpHeaders: \"OTEL_EXPORTER_OTLP_HEADERS\",\n  otelServiceName: \"OTEL_SERVICE_NAME\",\n  otelServiceVersion: \"OTEL_SERVICE_VERSION\",\n  otelResourceAttributes: \"OTEL_RESOURCE_ATTRIBUTES\",\n} as const;\n\n/** Defaults for optional platform-owned runtime configuration. */\nexport const PlatformRuntimeDefault = {\n  httpPort: 8080,\n  stateTable: \"_airfoil_connectors_state\",\n} as const;\n\nconst platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));\n\n/** @internal */\nexport const isPlatformRuntimeKey = (key: string): boolean => HashSet.has(platformRuntimeKeys, key);\n"],"mappings":";;;AAGA,MAAa,qBAAqB;CAChC,YAAY;CACZ,UAAU;CACV,eAAe;CACf,qBAAqB;CACrB,YAAY;CACZ,0BAA0B;CAC1B,WAAW;CACX,WAAW;CACX,SAAS;CACT,aAAa;CACb,0BAA0B;CAC1B,yBAAyB;CACzB,iBAAiB;CACjB,oBAAoB;CACpB,wBAAwB;AAC1B;;AAGA,MAAa,yBAAyB;CACpC,UAAU;CACV,YAAY;AACd;AAEA,MAAM,sBAAsB,QAAQ,aAAa,OAAO,OAAO,kBAAkB,CAAC;;AAGlF,MAAa,wBAAwB,QAAyB,QAAQ,IAAI,qBAAqB,GAAG"}

package/dist/demo/index.d.mts

@@ -1,2 +1,2 @@
-import { a as DemoDefinition, c as define, i as run, l as table, n as DemoTotals, o as DemoRows, r as RunOptions, s as DemoTable, u as DemoContext } from "../index-Dk7urvpF.mjs";
+import { a as DemoDefinition, c as define, i as run, l as table, n as DemoTotals, o as DemoRows, r as RunOptions, s as DemoTable, u as DemoContext } from "../index-CV6V6fyx.mjs";
 export { type DemoContext, type DemoDefinition, type DemoRows, type DemoTable, type DemoTotals, type RunOptions, define, run, table };

package/dist/demo/index.mjs

@@ -1,2 +1,2 @@
-import { i as table, n as run, r as define } from "../demo-B30Vlkk2.mjs";
+import { i as table, n as run, r as define } from "../demo-DqTJ7TWI.mjs";
 export { define, run, table };

package/dist/{demo-B30Vlkk2.mjs → demo-DqTJ7TWI.mjs}

@@ -1,9 +1,10 @@
 import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
 import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
-import { t as checkHasRepresentation } from "./table-V0tgUC5i.mjs";
+import { t as checkHasRepresentation } from "./table-CxM09xf7.mjs";
 import { t as Ingestor } from "./service-jWxxfEht.mjs";
-import { t as ingestBatch } from "./instrumented-Bhp7fyvh.mjs";
+import { t as ingestBatch } from "./instrumented-Bq1WX2L5.mjs";
 import { Array as Array$1, Duration, Effect, Option, Schema, Stream } from "effect";
+import humanFormat from "human-format";
 import { Faker, base, en } from "@faker-js/faker";
 //#region src/demo/definition.ts
 const table = (definition) => definition;
@@ -112,7 +113,6 @@ const empty = {
 	bytes: 0,
 	tables: {}
 };
-const batchRows = 500;
 const sum = (results) => ({
 	rows: results.reduce((total, result) => total + result.rows, 0),
 	bytes: results.reduce((total, result) => total + result.bytes, 0)
@@ -135,6 +135,10 @@ const add = (totals, written) => written.reduce((current, [name, result]) => ({
 */
 const run = Effect.fnUntraced(function* (demo, options) {
 	const ingestor = yield* Ingestor;
+	const batchSize = yield* Schema.decodeUnknownEffect(Schema.Int.check(Schema.isGreaterThan(0)))(options.batchSize ?? 500).pipe(Effect.mapError((cause) => new ConnectorError({
+		message: "Invalid demo batch size",
+		cause
+	})));
 	const bound = (table) => ingestor.resourceNames === void 0 || ingestor.resourceNames.includes(table.name);
 	const faker = new Faker({ locale: [en, base] });
 	faker.seed(options.seed);
@@ -147,34 +151,43 @@ const run = Effect.fnUntraced(function* (demo, options) {
 		fill,
 		decode: Schema.decodeUnknownEffect(Schema.toType(table.rowSchema))
 	}))));
-	const write = (tables, rowsOf) => Effect.forEach(tables, ({ table, fill, decode }) => Effect.forEach(rowsOf(table.name), (row) => decode(fill(context, row)).pipe(Effect.mapError((cause) => new ConnectorError({
-		message: `Invalid demo row for ${table.name}`,
-		cause
-	})))).pipe(Effect.flatMap((rows) => Effect.forEach(Array$1.chunksOf(rows, batchRows), (batch) => ingestBatch({
-		connector: demo.name,
-		resource: table.name,
-		source: "demo",
-		batch: { rows: batch }
-	}))), Effect.map((results) => [table.name, sum(results)])));
+	const write = (tables, rowsOf) => Effect.gen(function* () {
+		const prepared = yield* Effect.forEach(tables, ({ table, fill, decode }) => Effect.forEach(rowsOf(table.name), (row) => decode(fill(context, row)).pipe(Effect.mapError((cause) => new ConnectorError({
+			message: `Invalid demo row for ${table.name}`,
+			cause
+		})))).pipe(Effect.map((rows) => ({
+			table,
+			rows
+		}))));
+		return yield* Effect.forEach(prepared, ({ table, rows }) => Effect.forEach(Array$1.chunksOf(rows, batchSize), (rows) => ingestBatch({
+			connector: demo.name,
+			resource: table.name,
+			source: "demo",
+			batch: { rows }
+		}), { concurrency: 8 }).pipe(Effect.map((results) => [table.name, sum(results)])), { concurrency: 4 });
+	});
 	const setupTables = yield* prepare(demo.setup.tables);
 	const groupTables = yield* prepare(demo.group.tables);
 	const setup = demo.setup.rows(context);
 	const afterSetup = add(empty, yield* write(setupTables, (name) => setup[name] ?? []));
-	const totals = afterSetup.bytes >= options.bytes || groupTables.length === 0 ? afterSetup : yield* Stream.range(0, (demo.group.count ?? Infinity) - 1).pipe(Stream.grouped(options.groupsPerChunk ?? demo.group.groupsPerChunk ?? 100), Stream.mapEffect((indexes) => {
+	const totals = afterSetup.bytes >= options.bytes || groupTables.length === 0 ? afterSetup : yield* Stream.range(0, (demo.group.count ?? Infinity) - 1).pipe(Stream.grouped(options.groupsPerChunk ?? demo.group.groupsPerChunk ?? 1e3), Stream.mapEffect((indexes) => {
 		const groups = indexes.map((index) => demo.group.rows(index, context));
 		return write(groupTables, (name) => groups.flatMap((group) => group[name] ?? []));
 	}), Stream.scan(() => afterSetup, add), Stream.takeUntil((current) => current.bytes >= options.bytes), Stream.tap((current) => Effect.logInfo("demo progress").pipe(Effect.annotateLogs({
 		rows: current.rows,
-		bytes: current.bytes
+		size: humanFormat.bytes(current.bytes)
 	}))), Stream.runLast, Effect.map(Option.getOrElse(() => afterSetup)));
 	if (totals.bytes < options.bytes) yield* Effect.logWarning("The selected tables ran out of rows, so the run wrote less than the size asked for").pipe(Effect.annotateLogs({
-		bytes: totals.bytes,
-		target: options.bytes
+		size: humanFormat.bytes(totals.bytes),
+		target: humanFormat.bytes(options.bytes)
 	}));
 	yield* Effect.logInfo("demo done").pipe(Effect.annotateLogs({
 		rows: totals.rows,
-		bytes: totals.bytes,
-		tables: totals.tables
+		size: humanFormat.bytes(totals.bytes),
+		tables: Object.fromEntries(Object.entries(totals.tables).map(([name, result]) => [name, {
+			rows: result.rows,
+			size: humanFormat.bytes(result.bytes)
+		}]))
 	}));
 	return totals;
 });
@@ -188,4 +201,4 @@ var demo_exports = /* @__PURE__ */ __exportAll({
 //#endregion
 export { table as i, run as n, define as r, demo_exports as t };
 
-//# sourceMappingURL=demo-B30Vlkk2.mjs.map
+//# sourceMappingURL=demo-DqTJ7TWI.mjs.map

package/dist/demo-DqTJ7TWI.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"demo-DqTJ7TWI.mjs","names":["Arr"],"sources":["../src/demo/definition.ts","../src/demo/generate.ts","../src/demo/run.ts","../src/demo/index.ts"],"sourcesContent":["import type { Schema } from \"effect\";\n\nimport type { ResourceRequiredField, ResourceSchema } from \"../core/types\";\nimport type { DemoContext } from \"./generate\";\n\n/** A table a demo writes: the same name, row schema and version as its resource. */\nexport type DemoTable<\n  Name extends string = string,\n  S extends ResourceSchema = ResourceSchema,\n  Version extends string = string,\n> = {\n  readonly name: Name;\n  readonly rowSchema: S;\n  readonly version: Version;\n};\n\nexport const table = <\n  const Name extends string,\n  S extends ResourceSchema,\n  const Version extends ResourceRequiredField<S>,\n>(\n  definition: DemoTable<Name, S, Version>,\n): DemoTable<Name, S, Version> => definition;\n\n/** Rows by table. Required fields a row leaves out get placeholder values. */\nexport type DemoRows<Tables extends ReadonlyArray<DemoTable>> = {\n  readonly [Table in Tables[number] as Table[\"name\"]]?: ReadonlyArray<\n    Partial<Schema.Schema.Type<Table[\"rowSchema\"]>>\n  >;\n};\n\nexport type DemoDefinition<\n  Setup extends ReadonlyArray<DemoTable> = ReadonlyArray<DemoTable>,\n  Group extends ReadonlyArray<DemoTable> = ReadonlyArray<DemoTable>,\n> = {\n  readonly name: string;\n  /** Tables written once, before any group, such as a product catalog. */\n  readonly setup: {\n    readonly tables: Setup;\n    readonly rows: (context: DemoContext) => DemoRows<Setup>;\n  };\n  /**\n   * Tables for one related unit, such as a customer and their orders. The run\n   * adds groups until it reaches the size. Build keys from the index, so they\n   * are unique and other groups can point to them.\n   */\n  readonly group: {\n    readonly tables: Group;\n    readonly rows: (index: number, context: DemoContext) => DemoRows<Group>;\n    /**\n     * Groups written per chunk, 1000 by default. A run can go over the size by\n     * one chunk, so lower it when a group is big.\n     */\n    readonly groupsPerChunk?: number;\n    /**\n     * The number of groups there are, when the provider only keeps so much\n     * history. The run stops there, even below the size.\n     */\n    readonly count?: number;\n  };\n};\n\nexport const define = <\n  const Setup extends ReadonlyArray<DemoTable>,\n  const Group extends ReadonlyArray<DemoTable>,\n>(\n  definition: DemoDefinition<Setup, Group>,\n): DemoDefinition<Setup, Group> => definition;\n","import type { Faker } from \"@faker-js/faker\";\n\nimport { Duration, Effect, Schema, type SchemaRepresentation } from \"effect\";\n\nimport { ConnectorError } from \"../errors\";\nimport { checkHasRepresentation } from \"../iceberg/table\";\n\n/** What one run shares: the Faker and the time the run started. */\nexport type DemoContext = {\n  readonly faker: Faker;\n  readonly now: Date;\n};\n\n/** Completes a row: keeps what it has, and fills the required fields it lacks. */\nexport type RowFiller = (context: DemoContext, row: object) => Record<string, unknown>;\n\ntype Representation = SchemaRepresentation.Representation;\ntype Value = (context: DemoContext) => unknown;\n\nconst twoYears = Duration.toMillis(Duration.days(730));\n\nclass Unsupported extends Error {}\n\nconst unsupported = (path: string, what: string): never => {\n  throw new Unsupported(`${path}: ${what} is not supported in demo data`);\n};\n\n/**\n * Compiles a row schema into a filler for demo rows. It reads the same\n * representation as `compileTable`, so values fit the Iceberg types. It fills\n * only required fields the row lacks, so optional ones such as `_deleted` stay\n * out.\n */\nexport const compileRow = (schema: Schema.Top): Effect.Effect<RowFiller, ConnectorError> =>\n  Effect.try({\n    try: () => {\n      const { representation, references } = Schema.toRepresentation(Schema.toType(schema));\n\n      const resolve = (node: Representation, path: string): Representation => {\n        if (node._tag !== \"Reference\") return node;\n        const target = references[node.$ref];\n\n        return target === undefined ? unsupported(path, `reference ${node.$ref}`) : target;\n      };\n\n      const requiredFields = (node: Representation, path: string) => {\n        const resolved = resolve(node, path);\n\n        if (resolved._tag !== \"Objects\" || resolved.indexSignatures.length > 0) {\n          return unsupported(path, resolved._tag);\n        }\n\n        return resolved.propertySignatures\n          .filter((property) => !property.isOptional)\n          .map((property) => {\n            const name = String(property.name);\n            return [name, value(property.type, `${path}.${name}`)] as const;\n          });\n      };\n\n      const object = (node: Representation, path: string): Value => {\n        const fields = requiredFields(node, path);\n        return (context) => Object.fromEntries(fields.map(([name, make]) => [name, make(context)]));\n      };\n\n      const value = (node: Representation, path: string): Value => {\n        const resolved = resolve(node, path);\n\n        switch (resolved._tag) {\n          case \"String\":\n            return ({ faker }) => faker.word.words({ count: { min: 1, max: 3 } });\n          case \"Boolean\":\n            return ({ faker }) => faker.datatype.boolean();\n          case \"BigInt\":\n            return ({ faker }) => BigInt(faker.number.int({ max: 100_000 }));\n          case \"Number\":\n            return checkHasRepresentation(resolved.checks, \"effect/schema/isInt\")\n              ? ({ faker }) => faker.number.int({ max: 1000 })\n              : ({ faker }) => faker.number.float({ max: 1000, fractionDigits: 2 });\n          case \"Literal\": {\n            const literal = resolved.literal;\n            return () => literal;\n          }\n          case \"Union\": {\n            // Fill every field, so nullable unions get their value type.\n            const members = resolved.types.filter(\n              (member) => member._tag !== \"Null\" && member._tag !== \"Undefined\",\n            );\n\n            const literals = members.flatMap((member) =>\n              member._tag === \"Literal\" ? [member.literal] : [],\n            );\n\n            if (literals.length > 0 && literals.length === members.length) {\n              return ({ faker }) => faker.helpers.arrayElement(literals);\n            }\n\n            const [only] = members;\n\n            return only !== undefined && members.length === 1\n              ? value(only, path)\n              : unsupported(path, \"a union of several types\");\n          }\n          case \"Objects\":\n            return object(resolved, path);\n          case \"Arrays\": {\n            const [element] = resolved.rest;\n\n            if (resolved.elements.length > 0 || resolved.rest.length !== 1 || !element) {\n              return unsupported(path, \"a tuple\");\n            }\n\n            const item = value(element, `${path}[]`);\n\n            return (context) =>\n              Array.from({ length: context.faker.number.int({ max: 3 }) }, () => item(context));\n          }\n          case \"Declaration\":\n            return declaration(resolved, path);\n          case \"Suspend\":\n            return value(resolved.thunk, path);\n          default:\n            return unsupported(path, resolved._tag);\n        }\n      };\n\n      const declaration = (node: SchemaRepresentation.Declaration, path: string): Value => {\n        switch (node.representation?.id) {\n          case \"effect/schema/Date\":\n            return ({ faker, now }) =>\n              faker.date.between({ from: now.getTime() - twoYears, to: now });\n          case \"effect/schema/Uint8Array\":\n            return ({ faker }) => Uint8Array.from({ length: 8 }, () => faker.number.int(255));\n          case \"effect/schema/ReadonlyMap\": {\n            const [keyNode, valueNode] = node.typeParameters;\n            if (!keyNode || !valueNode) return unsupported(path, \"a map without key or value\");\n            const key = value(keyNode, `${path}.key`);\n            const item = value(valueNode, `${path}.value`);\n\n            return (context) =>\n              new Map(\n                Array.from({ length: context.faker.number.int({ max: 3 }) }, () => [\n                  key(context),\n                  item(context),\n                ]),\n              );\n          }\n          default:\n            return unsupported(path, node.representation?.id ?? \"an unnamed declaration\");\n        }\n      };\n\n      const fields = requiredFields(representation, \"$\");\n\n      return (context, row) => ({\n        ...Object.fromEntries(\n          fields.filter(([name]) => !(name in row)).map(([name, make]) => [name, make(context)]),\n        ),\n        ...row,\n      });\n    },\n    catch: (cause) =>\n      new ConnectorError({\n        message: cause instanceof Unsupported ? cause.message : \"Failed to compile demo rows\",\n        cause,\n      }),\n  });\n","import { Faker, base, en } from \"@faker-js/faker\";\nimport { Array as Arr, Effect, Option, Schema, Stream } from \"effect\";\nimport humanFormat from \"human-format\";\n\nimport type { DemoDefinition, DemoTable } from \"./definition\";\n\nimport { ConnectorError } from \"../errors\";\nimport { ingestBatch } from \"../ingestor/instrumented\";\nimport { Ingestor, type IngestResult } from \"../ingestor/service\";\nimport { compileRow, type DemoContext } from \"./generate\";\n\nexport type RunOptions = {\n  /** Stop once this many Arrow bytes are written. */\n  readonly bytes: number;\n  /** Omit for random data on each run. */\n  readonly seed?: number;\n  /** The time data is made around. Same seed, time and tables give the same rows. */\n  readonly now: Date;\n  /** Maximum rows per ingestion batch. Defaults to 500. Must be a positive integer. */\n  readonly batchSize?: number;\n  /** Groups per chunk. Overrides the demo's. */\n  readonly groupsPerChunk?: number;\n};\n\nexport type DemoTotals = IngestResult & {\n  readonly tables: Readonly<Record<string, IngestResult>>;\n};\n\nconst empty: DemoTotals = { rows: 0, bytes: 0, tables: {} };\n\nconst sum = (results: ReadonlyArray<IngestResult>): IngestResult => ({\n  rows: results.reduce((total, result) => total + result.rows, 0),\n  bytes: results.reduce((total, result) => total + result.bytes, 0),\n});\n\nconst add = (totals: DemoTotals, written: ReadonlyArray<readonly [string, IngestResult]>) =>\n  written.reduce<DemoTotals>(\n    (current, [name, result]) => ({\n      rows: current.rows + result.rows,\n      bytes: current.bytes + result.bytes,\n      tables: {\n        ...current.tables,\n        [name]: {\n          rows: (current.tables[name]?.rows ?? 0) + result.rows,\n          bytes: (current.tables[name]?.bytes ?? 0) + result.bytes,\n        },\n      },\n    }),\n    totals,\n  );\n\n/**\n * Writes the setup rows, then groups, until it reaches the size or the demo's\n * count. It can go over the size by one chunk. Only bound tables are\n * written; keys still point to the others.\n */\nexport const run = Effect.fnUntraced(function* (demo: DemoDefinition, options: RunOptions) {\n  const ingestor = yield* Ingestor;\n  // Wide rows may need a lower batch size to fit the engine's limits.\n  const batchSize = yield* Schema.decodeUnknownEffect(Schema.Int.check(Schema.isGreaterThan(0)))(\n    options.batchSize ?? 500,\n  ).pipe(\n    Effect.mapError((cause) => new ConnectorError({ message: \"Invalid demo batch size\", cause })),\n  );\n\n  const bound = (table: DemoTable) =>\n    ingestor.resourceNames === undefined || ingestor.resourceNames.includes(table.name);\n\n  // A fresh Faker per run keeps explicit seeds repeatable.\n  const faker = new Faker({ locale: [en, base] });\n  faker.seed(options.seed);\n  const context: DemoContext = { faker, now: options.now };\n\n  const prepare = (tables: ReadonlyArray<DemoTable>) =>\n    Effect.forEach(tables.filter(bound), (table) =>\n      compileRow(table.rowSchema).pipe(\n        Effect.map((fill) => ({\n          table,\n          fill,\n          decode: Schema.decodeUnknownEffect(Schema.toType(table.rowSchema)),\n        })),\n      ),\n    );\n\n  const write = (\n    tables: Effect.Success<ReturnType<typeof prepare>>,\n    rowsOf: (name: string) => ReadonlyArray<object>,\n  ) =>\n    Effect.gen(function* () {\n      // Fill rows in table order before concurrent writes, keeping Faker deterministic.\n      const prepared = yield* Effect.forEach(tables, ({ table, fill, decode }) =>\n        Effect.forEach(rowsOf(table.name), (row) =>\n          decode(fill(context, row)).pipe(\n            Effect.mapError(\n              (cause) =>\n                new ConnectorError({ message: `Invalid demo row for ${table.name}`, cause }),\n            ),\n          ),\n        ).pipe(Effect.map((rows) => ({ table, rows }))),\n      );\n\n      // Share durable flushes with bounded parallel writes.\n      return yield* Effect.forEach(\n        prepared,\n        ({ table, rows }) =>\n          Effect.forEach(\n            Arr.chunksOf(rows, batchSize),\n            (rows) =>\n              ingestBatch({\n                connector: demo.name,\n                resource: table.name,\n                source: \"demo\",\n                batch: { rows },\n              }),\n            { concurrency: 8 },\n          ).pipe(Effect.map((results) => [table.name, sum(results)] as const)),\n        { concurrency: 4 },\n      );\n    });\n\n  const setupTables = yield* prepare(demo.setup.tables);\n  const groupTables = yield* prepare(demo.group.tables);\n\n  const setup = demo.setup.rows(context);\n  const afterSetup = add(empty, yield* write(setupTables, (name) => setup[name] ?? []));\n\n  const totals =\n    afterSetup.bytes >= options.bytes || groupTables.length === 0\n      ? afterSetup\n      : yield* Stream.range(0, (demo.group.count ?? Infinity) - 1).pipe(\n          Stream.grouped(options.groupsPerChunk ?? demo.group.groupsPerChunk ?? 1000),\n          Stream.mapEffect((indexes) => {\n            const groups = indexes.map((index) => demo.group.rows(index, context));\n            return write(groupTables, (name) => groups.flatMap((group) => group[name] ?? []));\n          }),\n          Stream.scan(() => afterSetup, add),\n          Stream.takeUntil((current) => current.bytes >= options.bytes),\n          Stream.tap((current) =>\n            Effect.logInfo(\"demo progress\").pipe(\n              Effect.annotateLogs({ rows: current.rows, size: humanFormat.bytes(current.bytes) }),\n            ),\n          ),\n          Stream.runLast,\n          Effect.map(Option.getOrElse(() => afterSetup)),\n        );\n\n  if (totals.bytes < options.bytes) {\n    yield* Effect.logWarning(\n      \"The selected tables ran out of rows, so the run wrote less than the size asked for\",\n    ).pipe(\n      Effect.annotateLogs({\n        size: humanFormat.bytes(totals.bytes),\n        target: humanFormat.bytes(options.bytes),\n      }),\n    );\n  }\n\n  yield* Effect.logInfo(\"demo done\").pipe(\n    Effect.annotateLogs({\n      rows: totals.rows,\n      size: humanFormat.bytes(totals.bytes),\n      tables: Object.fromEntries(\n        Object.entries(totals.tables).map(([name, result]) => [\n          name,\n          { rows: result.rows, size: humanFormat.bytes(result.bytes) },\n        ]),\n      ),\n    }),\n  );\n\n  return totals;\n});\n","export { type DemoContext } from \"./generate\";\nexport { define, table, type DemoDefinition, type DemoRows, type DemoTable } from \"./definition\";\nexport { run, type DemoTotals, type RunOptions } from \"./run\";\n"],"mappings":";;;;;;;;;AAgBA,MAAa,SAKX,eACgC;AAwClC,MAAa,UAIX,eACiC;;;AChDnC,MAAM,WAAW,SAAS,SAAS,SAAS,KAAK,GAAG,CAAC;AAErD,IAAM,cAAN,cAA0B,MAAM,CAAC;AAEjC,MAAM,eAAe,MAAc,SAAwB;CACzD,MAAM,IAAI,YAAY,GAAG,KAAK,IAAI,KAAK,+BAA+B;AACxE;;;;;;;AAQA,MAAa,cAAc,WACzB,OAAO,IAAI;CACT,WAAW;EACT,MAAM,EAAE,gBAAgB,eAAe,OAAO,iBAAiB,OAAO,OAAO,MAAM,CAAC;EAEpF,MAAM,WAAW,MAAsB,SAAiC;GACtE,IAAI,KAAK,SAAS,aAAa,OAAO;GACtC,MAAM,SAAS,WAAW,KAAK;GAE/B,OAAO,WAAW,KAAA,IAAY,YAAY,MAAM,aAAa,KAAK,MAAM,IAAI;EAC9E;EAEA,MAAM,kBAAkB,MAAsB,SAAiB;GAC7D,MAAM,WAAW,QAAQ,MAAM,IAAI;GAEnC,IAAI,SAAS,SAAS,aAAa,SAAS,gBAAgB,SAAS,GACnE,OAAO,YAAY,MAAM,SAAS,IAAI;GAGxC,OAAO,SAAS,mBACb,QAAQ,aAAa,CAAC,SAAS,UAAU,CAAC,CAC1C,KAAK,aAAa;IACjB,MAAM,OAAO,OAAO,SAAS,IAAI;IACjC,OAAO,CAAC,MAAM,MAAM,SAAS,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;GACvD,CAAC;EACL;EAEA,MAAM,UAAU,MAAsB,SAAwB;GAC5D,MAAM,SAAS,eAAe,MAAM,IAAI;GACxC,QAAQ,YAAY,OAAO,YAAY,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC;EAC5F;EAEA,MAAM,SAAS,MAAsB,SAAwB;GAC3D,MAAM,WAAW,QAAQ,MAAM,IAAI;GAEnC,QAAQ,SAAS,MAAjB;IACE,KAAK,UACH,QAAQ,EAAE,YAAY,MAAM,KAAK,MAAM,EAAE,OAAO;KAAE,KAAK;KAAG,KAAK;IAAE,EAAE,CAAC;IACtE,KAAK,WACH,QAAQ,EAAE,YAAY,MAAM,SAAS,QAAQ;IAC/C,KAAK,UACH,QAAQ,EAAE,YAAY,OAAO,MAAM,OAAO,IAAI,EAAE,KAAK,IAAQ,CAAC,CAAC;IACjE,KAAK,UACH,OAAO,uBAAuB,SAAS,QAAQ,qBAAqB,KAC/D,EAAE,YAAY,MAAM,OAAO,IAAI,EAAE,KAAK,IAAK,CAAC,KAC5C,EAAE,YAAY,MAAM,OAAO,MAAM;KAAE,KAAK;KAAM,gBAAgB;IAAE,CAAC;IACxE,KAAK,WAAW;KACd,MAAM,UAAU,SAAS;KACzB,aAAa;IACf;IACA,KAAK,SAAS;KAEZ,MAAM,UAAU,SAAS,MAAM,QAC5B,WAAW,OAAO,SAAS,UAAU,OAAO,SAAS,WACxD;KAEA,MAAM,WAAW,QAAQ,SAAS,WAChC,OAAO,SAAS,YAAY,CAAC,OAAO,OAAO,IAAI,CAAC,CAClD;KAEA,IAAI,SAAS,SAAS,KAAK,SAAS,WAAW,QAAQ,QACrD,QAAQ,EAAE,YAAY,MAAM,QAAQ,aAAa,QAAQ;KAG3D,MAAM,CAAC,QAAQ;KAEf,OAAO,SAAS,KAAA,KAAa,QAAQ,WAAW,IAC5C,MAAM,MAAM,IAAI,IAChB,YAAY,MAAM,0BAA0B;IAClD;IACA,KAAK,WACH,OAAO,OAAO,UAAU,IAAI;IAC9B,KAAK,UAAU;KACb,MAAM,CAAC,WAAW,SAAS;KAE3B,IAAI,SAAS,SAAS,SAAS,KAAK,SAAS,KAAK,WAAW,KAAK,CAAC,SACjE,OAAO,YAAY,MAAM,SAAS;KAGpC,MAAM,OAAO,MAAM,SAAS,GAAG,KAAK,GAAG;KAEvC,QAAQ,YACN,MAAM,KAAK,EAAE,QAAQ,QAAQ,MAAM,OAAO,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,KAAK,OAAO,CAAC;IACpF;IACA,KAAK,eACH,OAAO,YAAY,UAAU,IAAI;IACnC,KAAK,WACH,OAAO,MAAM,SAAS,OAAO,IAAI;IACnC,SACE,OAAO,YAAY,MAAM,SAAS,IAAI;GAC1C;EACF;EAEA,MAAM,eAAe,MAAwC,SAAwB;GACnF,QAAQ,KAAK,gBAAgB,IAA7B;IACE,KAAK,sBACH,QAAQ,EAAE,OAAO,UACf,MAAM,KAAK,QAAQ;KAAE,MAAM,IAAI,QAAQ,IAAI;KAAU,IAAI;IAAI,CAAC;IAClE,KAAK,4BACH,QAAQ,EAAE,YAAY,WAAW,KAAK,EAAE,QAAQ,EAAE,SAAS,MAAM,OAAO,IAAI,GAAG,CAAC;IAClF,KAAK,6BAA6B;KAChC,MAAM,CAAC,SAAS,aAAa,KAAK;KAClC,IAAI,CAAC,WAAW,CAAC,WAAW,OAAO,YAAY,MAAM,4BAA4B;KACjF,MAAM,MAAM,MAAM,SAAS,GAAG,KAAK,KAAK;KACxC,MAAM,OAAO,MAAM,WAAW,GAAG,KAAK,OAAO;KAE7C,QAAQ,YACN,IAAI,IACF,MAAM,KAAK,EAAE,QAAQ,QAAQ,MAAM,OAAO,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,CACjE,IAAI,OAAO,GACX,KAAK,OAAO,CACd,CAAC,CACH;IACJ;IACA,SACE,OAAO,YAAY,MAAM,KAAK,gBAAgB,MAAM,wBAAwB;GAChF;EACF;EAEA,MAAM,SAAS,eAAe,gBAAgB,GAAG;EAEjD,QAAQ,SAAS,SAAS;GACxB,GAAG,OAAO,YACR,OAAO,QAAQ,CAAC,UAAU,EAAE,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,UAAU,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CACvF;GACA,GAAG;EACL;CACF;CACA,QAAQ,UACN,IAAI,eAAe;EACjB,SAAS,iBAAiB,cAAc,MAAM,UAAU;EACxD;CACF,CAAC;AACL,CAAC;;;AC1IH,MAAM,QAAoB;CAAE,MAAM;CAAG,OAAO;CAAG,QAAQ,CAAC;AAAE;AAE1D,MAAM,OAAO,aAAwD;CACnE,MAAM,QAAQ,QAAQ,OAAO,WAAW,QAAQ,OAAO,MAAM,CAAC;CAC9D,OAAO,QAAQ,QAAQ,OAAO,WAAW,QAAQ,OAAO,OAAO,CAAC;AAClE;AAEA,MAAM,OAAO,QAAoB,YAC/B,QAAQ,QACL,SAAS,CAAC,MAAM,aAAa;CAC5B,MAAM,QAAQ,OAAO,OAAO;CAC5B,OAAO,QAAQ,QAAQ,OAAO;CAC9B,QAAQ;EACN,GAAG,QAAQ;GACV,OAAO;GACN,OAAO,QAAQ,OAAO,KAAK,EAAE,QAAQ,KAAK,OAAO;GACjD,QAAQ,QAAQ,OAAO,KAAK,EAAE,SAAS,KAAK,OAAO;EACrD;CACF;AACF,IACA,MACF;;;;;;AAOF,MAAa,MAAM,OAAO,WAAW,WAAW,MAAsB,SAAqB;CACzF,MAAM,WAAW,OAAO;CAExB,MAAM,YAAY,OAAO,OAAO,oBAAoB,OAAO,IAAI,MAAM,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,CAC5F,QAAQ,aAAa,GACvB,CAAC,CAAC,KACA,OAAO,UAAU,UAAU,IAAI,eAAe;EAAE,SAAS;EAA2B;CAAM,CAAC,CAAC,CAC9F;CAEA,MAAM,SAAS,UACb,SAAS,kBAAkB,KAAA,KAAa,SAAS,cAAc,SAAS,MAAM,IAAI;CAGpF,MAAM,QAAQ,IAAI,MAAM,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;CAC9C,MAAM,KAAK,QAAQ,IAAI;CACvB,MAAM,UAAuB;EAAE;EAAO,KAAK,QAAQ;CAAI;CAEvD,MAAM,WAAW,WACf,OAAO,QAAQ,OAAO,OAAO,KAAK,IAAI,UACpC,WAAW,MAAM,SAAS,CAAC,CAAC,KAC1B,OAAO,KAAK,UAAU;EACpB;EACA;EACA,QAAQ,OAAO,oBAAoB,OAAO,OAAO,MAAM,SAAS,CAAC;CACnE,EAAE,CACJ,CACF;CAEF,MAAM,SACJ,QACA,WAEA,OAAO,IAAI,aAAa;EAEtB,MAAM,WAAW,OAAO,OAAO,QAAQ,SAAS,EAAE,OAAO,MAAM,aAC7D,OAAO,QAAQ,OAAO,MAAM,IAAI,IAAI,QAClC,OAAO,KAAK,SAAS,GAAG,CAAC,CAAC,CAAC,KACzB,OAAO,UACJ,UACC,IAAI,eAAe;GAAE,SAAS,wBAAwB,MAAM;GAAQ;EAAM,CAAC,CAC/E,CACF,CACF,CAAC,CAAC,KAAK,OAAO,KAAK,UAAU;GAAE;GAAO;EAAK,EAAE,CAAC,CAChD;EAGA,OAAO,OAAO,OAAO,QACnB,WACC,EAAE,OAAO,WACR,OAAO,QACLA,QAAI,SAAS,MAAM,SAAS,IAC3B,SACC,YAAY;GACV,WAAW,KAAK;GAChB,UAAU,MAAM;GAChB,QAAQ;GACR,OAAO,EAAE,KAAK;EAChB,CAAC,GACH,EAAE,aAAa,EAAE,CACnB,CAAC,CAAC,KAAK,OAAO,KAAK,YAAY,CAAC,MAAM,MAAM,IAAI,OAAO,CAAC,CAAU,CAAC,GACrE,EAAE,aAAa,EAAE,CACnB;CACF,CAAC;CAEH,MAAM,cAAc,OAAO,QAAQ,KAAK,MAAM,MAAM;CACpD,MAAM,cAAc,OAAO,QAAQ,KAAK,MAAM,MAAM;CAEpD,MAAM,QAAQ,KAAK,MAAM,KAAK,OAAO;CACrC,MAAM,aAAa,IAAI,OAAO,OAAO,MAAM,cAAc,SAAS,MAAM,SAAS,CAAC,CAAC,CAAC;CAEpF,MAAM,SACJ,WAAW,SAAS,QAAQ,SAAS,YAAY,WAAW,IACxD,aACA,OAAO,OAAO,MAAM,IAAI,KAAK,MAAM,SAAS,YAAY,CAAC,CAAC,CAAC,KACzD,OAAO,QAAQ,QAAQ,kBAAkB,KAAK,MAAM,kBAAkB,GAAI,GAC1E,OAAO,WAAW,YAAY;EAC5B,MAAM,SAAS,QAAQ,KAAK,UAAU,KAAK,MAAM,KAAK,OAAO,OAAO,CAAC;EACrE,OAAO,MAAM,cAAc,SAAS,OAAO,SAAS,UAAU,MAAM,SAAS,CAAC,CAAC,CAAC;CAClF,CAAC,GACD,OAAO,WAAW,YAAY,GAAG,GACjC,OAAO,WAAW,YAAY,QAAQ,SAAS,QAAQ,KAAK,GAC5D,OAAO,KAAK,YACV,OAAO,QAAQ,eAAe,CAAC,CAAC,KAC9B,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM,MAAM,YAAY,MAAM,QAAQ,KAAK;CAAE,CAAC,CACpF,CACF,GACA,OAAO,SACP,OAAO,IAAI,OAAO,gBAAgB,UAAU,CAAC,CAC/C;CAEN,IAAI,OAAO,QAAQ,QAAQ,OACzB,OAAO,OAAO,WACZ,oFACF,CAAC,CAAC,KACA,OAAO,aAAa;EAClB,MAAM,YAAY,MAAM,OAAO,KAAK;EACpC,QAAQ,YAAY,MAAM,QAAQ,KAAK;CACzC,CAAC,CACH;CAGF,OAAO,OAAO,QAAQ,WAAW,CAAC,CAAC,KACjC,OAAO,aAAa;EAClB,MAAM,OAAO;EACb,MAAM,YAAY,MAAM,OAAO,KAAK;EACpC,QAAQ,OAAO,YACb,OAAO,QAAQ,OAAO,MAAM,CAAC,CAAC,KAAK,CAAC,MAAM,YAAY,CACpD,MACA;GAAE,MAAM,OAAO;GAAM,MAAM,YAAY,MAAM,OAAO,KAAK;EAAE,CAC7D,CAAC,CACH;CACF,CAAC,CACH;CAEA,OAAO;AACT,CAAC"}

package/dist/iceberg/index.d.mts

@@ -1,3 +1,3 @@
-import { a as ids, i as identifier, r as field } from "../annotations-BE74YmAO.mjs";
-import { i as IcebergSchemaError, n as CreateTableCommitOptions, r as makeCreateTableCommitRequest } from "../index-DJhDp4Xc.mjs";
-export { type CreateTableCommitOptions, IcebergSchemaError, field, identifier, ids, makeCreateTableCommitRequest };
+import { r as identifier } from "../annotations-vzSzXtr-.mjs";
+import { i as IcebergSchemaError, n as CreateTableOptions, r as makeCreateTableRequest } from "../index-BTAbexBp.mjs";
+export { type CreateTableOptions, IcebergSchemaError, identifier, makeCreateTableRequest };

package/dist/iceberg/index.mjs

@@ -1,3 +1,3 @@
-import { a as IcebergSchemaError, i as makeCreateTableCommitRequest } from "../table-V0tgUC5i.mjs";
-import { i as ids, n as field, r as identifier } from "../iceberg-D6L8I6vA.mjs";
-export { IcebergSchemaError, field, identifier, ids, makeCreateTableCommitRequest };
+import { a as IcebergSchemaError, i as makeCreateTableRequest } from "../table-CxM09xf7.mjs";
+import { n as identifier } from "../iceberg-Cj2PwlNm.mjs";
+export { IcebergSchemaError, identifier, makeCreateTableRequest };

package/dist/iceberg-Cj2PwlNm.mjs

@@ -0,0 +1,30 @@
+import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
+import { a as IcebergSchemaError, i as makeCreateTableRequest } from "./table-CxM09xf7.mjs";
+import "effect";
+//#region src/iceberg/annotations.ts
+/**
+* Marks the fields that identify a row. The engine upserts rows by them.
+*
+* Put it last. A later `.annotate()` keeps the runtime annotation, but the type
+* loses the key, so `Resource.entity` stops compiling.
+*
+* @example
+* ```ts
+* const Post = Schema.Struct({
+*   id: Schema.String,
+*   updatedAt: Schema.String,
+* }).pipe(Iceberg.identifier("id"));
+* ```
+*/
+const identifier = (...keys) => (schema) => schema.annotate({ icebergIdentifierFields: keys });
+//#endregion
+//#region src/iceberg/index.ts
+var iceberg_exports = /* @__PURE__ */ __exportAll({
+	IcebergSchemaError: () => IcebergSchemaError,
+	identifier: () => identifier,
+	makeCreateTableRequest: () => makeCreateTableRequest
+});
+//#endregion
+export { identifier as n, iceberg_exports as t };
+
+//# sourceMappingURL=iceberg-Cj2PwlNm.mjs.map