@useairfoil/connector-kit 0.6.1 → 0.7.0

package/src/catalog/table.ts

@@ -6,30 +6,34 @@ import type {
   TableSchema,
 } from "iceberg-js";
 
-import * as Wings from "@useairfoil/wings";
+import { CatalogManager } from "@useairfoil/api";
 import { Effect, type Schema } from "effect";
 import { HttpClient } from "effect/http";
 import { getCurrentSchema } from "iceberg-js";
 
 import { ConnectorError } from "../errors";
 import { IcebergSchemaError } from "../iceberg/error";
-import { compileTable, makeCommitRequest } from "../iceberg/table";
+import { compileTable, makeCreateRequest } from "../iceberg/table";
+import { retryCatalogRequest, type TableRetryOptions } from "./retry";
 
 type Catalog = Effect.Success<
-  ReturnType<Wings.CatalogManager.CatalogManagerService["getIcebergCatalog"]>
+  ReturnType<CatalogManager.CatalogManagerService["getIcebergCatalog"]>
 >;
 
 type TableOptions = {
-  readonly catalog: Catalog;
+  readonly retry?: TableRetryOptions;
+  readonly catalog: Pick<Catalog, "loadTable" | "createTable" | "createNamespaceIfNotExists">;
   readonly identifier: TableIdentifier;
   readonly location?: string | undefined;
+  readonly properties?: Readonly<Record<string, string>> | undefined;
 };
 
-type WingsTableOptions = {
-  readonly wingsUri: string;
+type EngineTableOptions = {
+  readonly engineUri: string;
   readonly catalogId: string;
   readonly identifier: TableIdentifier;
   readonly location?: string | undefined;
+  readonly properties?: Readonly<Record<string, string>> | undefined;
 };
 
 const tableError = (cause: unknown): ConnectorError => {
@@ -58,21 +62,12 @@ const compareTypes = (
       if (actual.type !== "struct") return path;
       return compareFields(expected.fields, actual.fields, path);
     case "list":
-      if (
-        actual.type !== "list" ||
-        expected["element-id"] !== actual["element-id"] ||
-        expected["element-required"] !== actual["element-required"]
-      ) {
+      if (actual.type !== "list" || expected["element-required"] !== actual["element-required"]) {
         return path;
       }
       return compareTypes(expected.element, actual.element, `${path}.element`);
     case "map":
-      if (
-        actual.type !== "map" ||
-        expected["key-id"] !== actual["key-id"] ||
-        expected["value-id"] !== actual["value-id"] ||
-        expected["value-required"] !== actual["value-required"]
-      ) {
+      if (actual.type !== "map" || expected["value-required"] !== actual["value-required"]) {
         return path;
       }
       return (
@@ -93,7 +88,7 @@ const compareFields = (
     const fieldPath = `${path}.${field.name}`;
     const other = actual.find((candidate) => candidate.name === field.name);
 
-    if (!other || field.id !== other.id || field.required !== other.required) return fieldPath;
+    if (!other || field.required !== other.required) return fieldPath;
 
     const mismatch = compareTypes(field.type, other.type, fieldPath);
     if (mismatch) return mismatch;
@@ -107,6 +102,21 @@ const verifyTable = (
   Effect.try({
     try: () => {
       if (!Array.isArray(metadata.schemas)) throw new Error("Invalid table metadata");
+      if (metadata["format-version"] !== 3) {
+        throw new Error(
+          "Storage-engine tables must use Iceberg format v3. Migrate the table before ingesting.",
+        );
+      }
+      const partitionSpec = metadata["partition-specs"].find(
+        (spec) => spec["spec-id"] === metadata["default-spec-id"],
+      );
+      if (!partitionSpec) throw new Error("Table has no default partition spec");
+      if (partitionSpec.fields.length > 0) {
+        throw new Error("Storage-engine tables must be unpartitioned.");
+      }
+      if (metadata.properties?.["encryption.key-id"] !== undefined) {
+        throw new Error("Storage-engine tables must be unencrypted.");
+      }
       const actual = getCurrentSchema(metadata);
       if (!actual) throw new Error("Table has no current schema");
 
@@ -117,11 +127,17 @@ const verifyTable = (
         );
       }
 
-      const expectedIds = new Set(expected["identifier-field-ids"]);
-      const actualIds = new Set(actual["identifier-field-ids"]);
+      const identifierNames = (schema: TableSchema) =>
+        new Set(
+          schema.fields
+            .filter((field) => schema["identifier-field-ids"]?.includes(field.id))
+            .map((field) => field.name),
+        );
+      const expectedNames = identifierNames(expected);
+      const actualNames = identifierNames(actual);
       if (
-        expectedIds.size !== actualIds.size ||
-        [...expectedIds].some((id) => !actualIds.has(id))
+        expectedNames.size !== actualNames.size ||
+        [...expectedNames].some((name) => !actualNames.has(name))
       ) {
         throw new Error(
           "Table identifier fields do not match. Recreate the table or revert the schema change.",
@@ -137,35 +153,40 @@ const verifyTable = (
       }),
   });
 
-/** Creates a table with the IDs from its Effect schema, then checks the loaded schema. */
+/** Creates a table through the REST catalog, then checks the loaded schema. */
 export const createTable = (
   schema: Schema.Top,
   options: TableOptions,
 ): Effect.Effect<TableSchema, ConnectorError> =>
   Effect.gen(function* () {
     const table = yield* compileTable(schema);
-    yield* options.catalog.commitTable(
-      options.identifier,
-      makeCommitRequest(table, { location: options.location }),
+    yield* options.catalog.createTable(
+      { namespace: options.identifier.namespace },
+      makeCreateRequest(table, {
+        name: options.identifier.name,
+        location: options.location,
+        properties: options.properties,
+      }),
     );
 
     const metadata = yield* options.catalog.loadTable(options.identifier);
     return yield* verifyTable(metadata, table.schema);
   }).pipe(Effect.mapError(tableError));
 
-/** Creates a table through a Wings catalog. */
-export const createTableFromWings = (
+/** Creates a missing table through an engine catalog and verifies its schema. */
+export const ensureTableFromEngine = (
   schema: Schema.Top,
-  options: WingsTableOptions,
+  options: EngineTableOptions,
 ): Effect.Effect<TableSchema, ConnectorError, HttpClient.HttpClient> =>
   Effect.scoped(
     Effect.gen(function* () {
-      const manager = yield* Wings.CatalogManager.make({ baseUrl: options.wingsUri });
+      const manager = yield* CatalogManager.make({ engineUri: options.engineUri });
       const catalog = yield* manager.getIcebergCatalog(options.catalogId);
-      return yield* createTable(schema, {
+      return yield* ensureTable(schema, {
         catalog,
         identifier: options.identifier,
         location: options.location,
+        properties: options.properties,
       });
     }),
   ).pipe(Effect.mapError(tableError));
@@ -177,14 +198,29 @@ export const ensureTable = (
 ): Effect.Effect<TableSchema, ConnectorError> =>
   Effect.gen(function* () {
     const table = yield* compileTable(schema);
-    const metadata = yield* options.catalog.loadTable(options.identifier).pipe(
+    const request = <A>(
+      effect: Effect.Effect<A, Effect.Error<ReturnType<Catalog["loadConfig"]>>>,
+    ) => retryCatalogRequest(effect, options.retry);
+    const metadata = yield* request(options.catalog.loadTable(options.identifier)).pipe(
       Effect.catchIf(
         (error) => error.status === 404,
         () =>
-          options.catalog
-            .commitTable(
-              options.identifier,
-              makeCommitRequest(table, { location: options.location }),
+          request(
+            options.catalog.createNamespaceIfNotExists({ namespace: options.identifier.namespace }),
+          )
+            .pipe(
+              Effect.andThen(
+                request(
+                  options.catalog.createTable(
+                    { namespace: options.identifier.namespace },
+                    makeCreateRequest(table, {
+                      name: options.identifier.name,
+                      location: options.location,
+                      properties: options.properties,
+                    }),
+                  ),
+                ),
+              ),
             )
             .pipe(
               // Another instance may have created the table. Reload and check it.
@@ -192,7 +228,7 @@ export const ensureTable = (
                 (error) => error.status === 409 || error.status === 412,
                 () => Effect.void,
               ),
-              Effect.flatMap(() => options.catalog.loadTable(options.identifier)),
+              Effect.flatMap(() => request(options.catalog.loadTable(options.identifier))),
             ),
       ),
     );

package/src/connector-app/create-table.ts

@@ -1,10 +1,10 @@
 import { Effect, Option, type Schema } from "effect";
 import { Argument, CliError, Command, Flag } from "effect/cli";
 
-import { createTableFromWings } from "../catalog/table";
-import { wingsUri } from "../runtime-config/config";
+import { ensureTableFromEngine } from "../catalog/table";
+import { engineUri } from "../runtime-config/config";
 
-/** Creates the table command for a connector's row schemas. */
+/** Creates the table command; existing tables must match the connector's row schemas. */
 export const makeCreateTableCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>
   Command.make(
     "create-table",
@@ -35,13 +35,13 @@ export const makeCreateTableCommand = (schemas: Readonly<Record<string, Schema.T
           });
         }
 
-        const uri = yield* wingsUri;
-        yield* createTableFromWings(schema, {
-          wingsUri: uri.href,
+        const uri = yield* engineUri;
+        yield* ensureTableFromEngine(schema, {
+          engineUri: uri.href,
           catalogId: catalog,
           identifier: { namespace, name },
           location: Option.getOrUndefined(location),
         });
-        yield* Effect.log(`Created ${table}`);
+        yield* Effect.log(`Prepared ${table}`);
       }),
-  ).pipe(Command.withDescription("Create an Iceberg table for a connector resource"));
+  ).pipe(Command.withDescription("Create or verify an Iceberg table for a connector resource"));

package/src/connector-app/demo.ts

@@ -1,41 +1,46 @@
-import { ByteSize, DateTime, Effect, Option } from "effect";
+import { DateTime, Effect, Option, Schema } from "effect";
 import { Command, Flag } from "effect/cli";
+import humanFormat from "human-format";
 
 import { run, type DemoDefinition } from "../demo";
-import { layerWingsConfig } from "../ingestor/wings";
+import { layerArrowFlightConfig } from "../ingestor/arrow-flight";
 
-/** Reads sizes such as `512KiB`, `128MiB` or `1GiB`. */
-export const parseSize = (input: string): Option.Option<number> => {
-  if (!/^\d+(KiB|MiB|GiB)$/.test(input)) return Option.none();
-
-  return ByteSize.fromString(input).pipe(
-    Option.flatMap(ByteSize.toNumber),
-    Option.filter((bytes) => bytes > 0),
+/** Reads byte sizes such as `512kB`, `128MB` or `1.5GB`. */
+export const parseSize = (input: string): Option.Option<number> =>
+  Option.liftThrowable(() => humanFormat.parse.bytes(input))().pipe(
+    Option.filter((bytes) => Number.isSafeInteger(bytes) && bytes > 0),
   );
-};
 
 /**
- * Creates the command that writes the demo data to Wings, then exits.
- * It reads Wings and table bindings from Config, so provide `RuntimeConfig.layerHosted()`.
+ * Creates the command that writes demo data over Arrow Flight, then exits.
+ * It reads hosted config and table bindings, so provide `RuntimeConfig.layerHosted()`.
  */
 export const makeDemoCommand = (demo: DemoDefinition) =>
   Command.make(
     "demo",
     {
       size: Flag.String("size").pipe(
-        Flag.filterMap(parseSize, () => "a size like 512KiB, 128MiB or 1GiB"),
+        Flag.filterMap(parseSize, () => "a size like 512kB, 128MB or 1.5GB"),
+      ),
+      seed: Flag.Int("seed").pipe(Flag.optional),
+      batchSize: Flag.Int("batch-size").pipe(
+        Flag.withSchema(Schema.Int.check(Schema.isGreaterThan(0))),
+        Flag.withDescription("Maximum rows per ingestion batch (default: 500)"),
+        Flag.optional,
       ),
-      seed: Flag.Int("seed").pipe(Flag.withDefault(1)),
     },
-    ({ size, seed }) =>
+    ({ size, seed, batchSize }) =>
       Effect.gen(function* () {
         const now = yield* DateTime.nowAsDate;
 
-        yield* run(demo, { bytes: size, seed, now }).pipe(
+        yield* run(demo, {
+          bytes: size,
+          seed: Option.getOrUndefined(seed),
+          batchSize: Option.getOrUndefined(batchSize),
+          now,
+        }).pipe(
           Effect.provide(
-            layerWingsConfig({
-              resources: [...demo.setup.tables, ...demo.group.tables],
-            }),
+            layerArrowFlightConfig({ resources: [...demo.setup.tables, ...demo.group.tables] }),
           ),
         );
       }),

package/src/connector-app/index.ts

@@ -2,3 +2,4 @@ export { check, type CheckOptions, type CheckResult, type ResourceCheckResult }
 export { makeCreateTableCommand } from "./create-table";
 export { makeDemoCommand, parseSize } from "./demo";
 export { start, type App, type StartOptions } from "./start";
+export { makeSetupCommand } from "./setup";

package/src/connector-app/setup.ts

@@ -0,0 +1,69 @@
+import { CatalogManager } from "@useairfoil/api";
+import { Duration, Effect, Option, Schema } from "effect";
+import { Command, Flag } from "effect/cli";
+
+import { retryCatalogRequest } from "../catalog/retry";
+import { ensureTable } from "../catalog/table";
+import * as RuntimeConfig from "../runtime-config";
+
+/** Creates or verifies all bound tables without loading provider credentials. */
+export const makeSetupCommand = (schemas: Readonly<Record<string, Schema.Top>>) =>
+  Command.make(
+    "setup",
+    {
+      checkpointIntervalSec: Flag.Int("checkpoint-interval-sec").pipe(
+        Flag.withSchema(Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: 900 }))),
+        Flag.withDescription(
+          "Checkpoint interval for new tables in seconds (0 disables the interval delay)",
+        ),
+        Flag.optional,
+      ),
+      retryInterval: Flag.String("retry-interval").pipe(
+        Flag.withDefault("5 seconds"),
+        Flag.withSchema(Schema.DurationFromString),
+        Flag.filter(
+          (duration) => Duration.isPositive(duration) && Duration.isFinite(duration),
+          () => "a positive duration, such as 5 seconds",
+        ),
+        Flag.withDescription("Delay between transient catalog retries"),
+      ),
+      retries: Flag.Int("retries").pipe(
+        Flag.withDefault(10),
+        Flag.filter(
+          (value) => value >= 0,
+          () => "a non-negative number of retries",
+        ),
+        Flag.withDescription("Maximum retries per catalog request after the initial attempt"),
+      ),
+    },
+    ({ retryInterval, retries, checkpointIntervalSec }) =>
+      Effect.gen(function* () {
+        const source = {
+          resources: Object.entries(schemas).map(([name, rowSchema]) => ({ name, rowSchema })),
+        };
+        const tables = yield* RuntimeConfig.loadTableBindings(source);
+        const uri = yield* RuntimeConfig.engineUri;
+        const catalogId = yield* RuntimeConfig.catalog;
+        const manager = yield* CatalogManager.make({ engineUri: uri.href });
+        const retry = { interval: retryInterval, times: retries };
+        const properties = Option.match(checkpointIntervalSec, {
+          onNone: () => undefined,
+          onSome: (seconds) => ({ "airfoil.checkpoint.interval-sec": String(seconds) }),
+        });
+        const catalog = yield* retryCatalogRequest(manager.getIcebergCatalog(catalogId), retry);
+
+        // Sequential setup avoids bursting the catalog's table-creation quota.
+        for (const resource of source.resources) {
+          const binding = tables[resource.name];
+          if (!binding) continue;
+          yield* ensureTable(resource.rowSchema, {
+            catalog,
+            identifier: binding,
+            location: binding.location,
+            retry,
+            properties,
+          });
+          yield* Effect.logInfo(`Prepared ${[...binding.namespace, binding.name].join(".")}`);
+        }
+      }),
+  ).pipe(Command.withDescription("Create or verify the Iceberg tables in AIRFOIL_TABLE_BINDINGS"));

package/src/core/types.ts

@@ -96,7 +96,7 @@ export type ResourceDelete<S extends ResourceSchema, Version extends ResourceFie
   readonly _deleted: true;
 };
 
-/** A full row, or a delete. Wings replaces the whole row. */
+/** A full row, or a delete. The engine replaces the whole row. */
 export type ResourceUpdate<S extends ResourceSchema, Version extends ResourceField<S>> =
   | (ResourceRow<S> & { readonly _deleted?: boolean })
   | ResourceDelete<S, Version>;

package/src/demo/definition.ts

@@ -48,7 +48,7 @@ export type DemoDefinition<
     readonly tables: Group;
     readonly rows: (index: number, context: DemoContext) => DemoRows<Group>;
     /**
-     * Groups written per chunk, 100 by default. A run can go over the size by
+     * Groups written per chunk, 1000 by default. A run can go over the size by
      * one chunk, so lower it when a group is big.
      */
     readonly groupsPerChunk?: number;

package/src/demo/generate.ts

@@ -5,7 +5,7 @@ import { Duration, Effect, Schema, type SchemaRepresentation } from "effect";
 import { ConnectorError } from "../errors";
 import { checkHasRepresentation } from "../iceberg/table";
 
-/** What one run shares: the seeded Faker and the time the run started. */
+/** What one run shares: the Faker and the time the run started. */
 export type DemoContext = {
   readonly faker: Faker;
   readonly now: Date;

package/src/demo/run.ts

@@ -1,5 +1,6 @@
 import { Faker, base, en } from "@faker-js/faker";
 import { Array as Arr, Effect, Option, Schema, Stream } from "effect";
+import humanFormat from "human-format";
 
 import type { DemoDefinition, DemoTable } from "./definition";
 
@@ -11,9 +12,12 @@ import { compileRow, type DemoContext } from "./generate";
 export type RunOptions = {
   /** Stop once this many Arrow bytes are written. */
   readonly bytes: number;
-  readonly seed: number;
+  /** Omit for random data on each run. */
+  readonly seed?: number;
   /** The time data is made around. Same seed, time and tables give the same rows. */
   readonly now: Date;
+  /** Maximum rows per ingestion batch. Defaults to 500. Must be a positive integer. */
+  readonly batchSize?: number;
   /** Groups per chunk. Overrides the demo's. */
   readonly groupsPerChunk?: number;
 };
@@ -24,10 +28,6 @@ export type DemoTotals = IngestResult & {
 
 const empty: DemoTotals = { rows: 0, bytes: 0, tables: {} };
 
-// Wings rejects Flight messages over 4 MiB. 500 rows keeps a batch of the
-// widest demo rows well under that.
-const batchRows = 500;
-
 const sum = (results: ReadonlyArray<IngestResult>): IngestResult => ({
   rows: results.reduce((total, result) => total + result.rows, 0),
   bytes: results.reduce((total, result) => total + result.bytes, 0),
@@ -56,11 +56,17 @@ const add = (totals: DemoTotals, written: ReadonlyArray<readonly [string, Ingest
  */
 export const run = Effect.fnUntraced(function* (demo: DemoDefinition, options: RunOptions) {
   const ingestor = yield* Ingestor;
+  // Wide rows may need a lower batch size to fit the engine's limits.
+  const batchSize = yield* Schema.decodeUnknownEffect(Schema.Int.check(Schema.isGreaterThan(0)))(
+    options.batchSize ?? 500,
+  ).pipe(
+    Effect.mapError((cause) => new ConnectorError({ message: "Invalid demo batch size", cause })),
+  );
 
   const bound = (table: DemoTable) =>
     ingestor.resourceNames === undefined || ingestor.resourceNames.includes(table.name);
 
-  // A fresh Faker per run, so its sequence depends only on the seed.
+  // A fresh Faker per run keeps explicit seeds repeatable.
   const faker = new Faker({ locale: [en, base] });
   faker.seed(options.seed);
   const context: DemoContext = { faker, now: options.now };
@@ -80,27 +86,37 @@ export const run = Effect.fnUntraced(function* (demo: DemoDefinition, options: R
     tables: Effect.Success<ReturnType<typeof prepare>>,
     rowsOf: (name: string) => ReadonlyArray<object>,
   ) =>
-    Effect.forEach(tables, ({ table, fill, decode }) =>
-      Effect.forEach(rowsOf(table.name), (row) =>
-        decode(fill(context, row)).pipe(
-          Effect.mapError(
-            (cause) => new ConnectorError({ message: `Invalid demo row for ${table.name}`, cause }),
-          ),
-        ),
-      ).pipe(
-        Effect.flatMap((rows) =>
-          Effect.forEach(Arr.chunksOf(rows, batchRows), (batch) =>
-            ingestBatch({
-              connector: demo.name,
-              resource: table.name,
-              source: "demo",
-              batch: { rows: batch },
-            }),
+    Effect.gen(function* () {
+      // Fill rows in table order before concurrent writes, keeping Faker deterministic.
+      const prepared = yield* Effect.forEach(tables, ({ table, fill, decode }) =>
+        Effect.forEach(rowsOf(table.name), (row) =>
+          decode(fill(context, row)).pipe(
+            Effect.mapError(
+              (cause) =>
+                new ConnectorError({ message: `Invalid demo row for ${table.name}`, cause }),
+            ),
           ),
-        ),
-        Effect.map((results) => [table.name, sum(results)] as const),
-      ),
-    );
+        ).pipe(Effect.map((rows) => ({ table, rows }))),
+      );
+
+      // Share durable flushes with bounded parallel writes.
+      return yield* Effect.forEach(
+        prepared,
+        ({ table, rows }) =>
+          Effect.forEach(
+            Arr.chunksOf(rows, batchSize),
+            (rows) =>
+              ingestBatch({
+                connector: demo.name,
+                resource: table.name,
+                source: "demo",
+                batch: { rows },
+              }),
+            { concurrency: 8 },
+          ).pipe(Effect.map((results) => [table.name, sum(results)] as const)),
+        { concurrency: 4 },
+      );
+    });
 
   const setupTables = yield* prepare(demo.setup.tables);
   const groupTables = yield* prepare(demo.group.tables);
@@ -112,7 +128,7 @@ export const run = Effect.fnUntraced(function* (demo: DemoDefinition, options: R
     afterSetup.bytes >= options.bytes || groupTables.length === 0
       ? afterSetup
       : yield* Stream.range(0, (demo.group.count ?? Infinity) - 1).pipe(
-          Stream.grouped(options.groupsPerChunk ?? demo.group.groupsPerChunk ?? 100),
+          Stream.grouped(options.groupsPerChunk ?? demo.group.groupsPerChunk ?? 1000),
           Stream.mapEffect((indexes) => {
             const groups = indexes.map((index) => demo.group.rows(index, context));
             return write(groupTables, (name) => groups.flatMap((group) => group[name] ?? []));
@@ -121,7 +137,7 @@ export const run = Effect.fnUntraced(function* (demo: DemoDefinition, options: R
           Stream.takeUntil((current) => current.bytes >= options.bytes),
           Stream.tap((current) =>
             Effect.logInfo("demo progress").pipe(
-              Effect.annotateLogs({ rows: current.rows, bytes: current.bytes }),
+              Effect.annotateLogs({ rows: current.rows, size: humanFormat.bytes(current.bytes) }),
             ),
           ),
           Stream.runLast,
@@ -131,11 +147,25 @@ export const run = Effect.fnUntraced(function* (demo: DemoDefinition, options: R
   if (totals.bytes < options.bytes) {
     yield* Effect.logWarning(
       "The selected tables ran out of rows, so the run wrote less than the size asked for",
-    ).pipe(Effect.annotateLogs({ bytes: totals.bytes, target: options.bytes }));
+    ).pipe(
+      Effect.annotateLogs({
+        size: humanFormat.bytes(totals.bytes),
+        target: humanFormat.bytes(options.bytes),
+      }),
+    );
   }
 
   yield* Effect.logInfo("demo done").pipe(
-    Effect.annotateLogs({ rows: totals.rows, bytes: totals.bytes, tables: totals.tables }),
+    Effect.annotateLogs({
+      rows: totals.rows,
+      size: humanFormat.bytes(totals.bytes),
+      tables: Object.fromEntries(
+        Object.entries(totals.tables).map(([name, result]) => [
+          name,
+          { rows: result.rows, size: humanFormat.bytes(result.bytes) },
+        ]),
+      ),
+    }),
   );
 
   return totals;

package/src/iceberg/annotations.ts

@@ -3,24 +3,11 @@ import { Schema } from "effect";
 declare module "effect/Schema" {
   namespace Annotations {
     interface Annotations {
-      readonly fieldId?: number | undefined;
       readonly icebergIdentifierFields?: ReadonlyArray<string> | undefined;
     }
   }
 }
 
-/** Sets a field's Iceberg ID, and any other annotations. */
-export const field =
-  <A = unknown>(fieldId: number, annotations: Schema.Annotations.Key<A> = {}) =>
-  <S extends Schema.Schema<A>>(schema: S): S["Rebuild"] =>
-    schema.pipe(Schema.annotateKey({ ...annotations, fieldId }));
-
-/** Adds an offset to a base ID for reusable schemas. */
-export const ids =
-  (baseId: number) =>
-  (offset: number): number =>
-    baseId + offset;
-
 type KnownField<S extends Schema.Top> = keyof S["Type"] & string;
 
 /** A row field that cannot be null or undefined. */
@@ -40,7 +27,7 @@ export interface Identified<Key extends string> {
 }
 
 /**
- * Marks the fields that identify a row. Wings upserts rows by them.
+ * Marks the fields that identify a row. The engine upserts rows by them.
  *
  * Put it last. A later `.annotate()` keeps the runtime annotation, but the type
  * loses the key, so `Resource.entity` stops compiling.
@@ -48,8 +35,8 @@ export interface Identified<Key extends string> {
  * @example
  * ```ts
  * const Post = Schema.Struct({
- *   id: Schema.String.pipe(Iceberg.field(1)),
- *   updatedAt: Schema.String.pipe(Iceberg.field(2)),
+ *   id: Schema.String,
+ *   updatedAt: Schema.String,
  * }).pipe(Iceberg.identifier("id"));
  * ```
  */

package/src/iceberg/index.ts

@@ -1,3 +1,3 @@
-export { field, identifier, ids } from "./annotations";
+export { identifier } from "./annotations";
 export { IcebergSchemaError } from "./error";
-export { makeCreateTableCommitRequest, type CreateTableCommitOptions } from "./table";
+export { makeCreateTableRequest, type CreateTableOptions } from "./table";