@useairfoil/flight 0.4.0 → 0.4.2

package/src/proto/any.ts

@@ -242,24 +242,17 @@ function base64FromBytes(arr: Uint8Array): string {
 
 type Builtin = Date | Function | Uint8Array | string | number | boolean | bigint | undefined;
 
-export type DeepPartial<T> = T extends Builtin
-  ? T
-  : T extends globalThis.Array<infer U>
-    ? globalThis.Array<DeepPartial<U>>
-    : T extends ReadonlyArray<infer U>
-      ? ReadonlyArray<DeepPartial<U>>
-      : T extends { readonly $case: string }
-        ? { [K in keyof Omit<T, "$case">]?: DeepPartial<T[K]> } & { readonly $case: T["$case"] }
-        : T extends {}
-          ? { [K in Exclude<keyof T, "$type">]?: DeepPartial<T[K]> }
-          : Partial<T>;
+export type DeepPartial<T> = T extends Builtin ? T
+  : T extends globalThis.Array<infer U> ? globalThis.Array<DeepPartial<U>>
+  : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>>
+  : T extends { readonly $case: string }
+    ? { [K in keyof Omit<T, "$case">]?: DeepPartial<T[K]> } & { readonly $case: T["$case"] }
+  : T extends {} ? { [K in Exclude<keyof T, "$type">]?: DeepPartial<T[K]> }
+  : Partial<T>;
 
 type KeysOfUnion<T> = T extends T ? keyof T : never;
-export type Exact<P, I extends P> = P extends Builtin
-  ? P
-  : P & { [K in keyof P]: Exact<P[K], I[K]> } & {
-      [K in Exclude<keyof I, KeysOfUnion<P> | "$type">]: never;
-    };
+export type Exact<P, I extends P> = P extends Builtin ? P
+  : P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P> | "$type">]: never };
 
 function isSet(value: any): boolean {
   return value !== null && value !== undefined;

package/src/proto/google/protobuf/descriptor.ts

@@ -34,6 +34,8 @@ PROTO3 = 999,
  */
 2023 = 1000,
 2024 = 1001,
+/** UNSTABLE - A placeholder edition for developing and testing unscheduled features. */
+UNSTABLE = 9999,
 /**
  * 1_TEST_ONLY - Placeholder editions for testing feature resolution.  These should not be
  * used or relied on outside of tests.
@@ -72,6 +74,9 @@ case 1000:
 case 1001:
       case "EDITION_2024":
         return Edition.2024;
+case 9999:
+      case "EDITION_UNSTABLE":
+        return Edition.UNSTABLE;
 case 1:
       case "EDITION_1_TEST_ONLY":
         return Edition.1_TEST_ONLY;
@@ -105,6 +110,7 @@ case Edition.PROTO2: return "EDITION_PROTO2";
 case Edition.PROTO3: return "EDITION_PROTO3";
 case Edition.2023: return "EDITION_2023";
 case Edition.2024: return "EDITION_2024";
+case Edition.UNSTABLE: return "EDITION_UNSTABLE";
 case Edition.1_TEST_ONLY: return "EDITION_1_TEST_ONLY";
 case Edition.2_TEST_ONLY: return "EDITION_2_TEST_ONLY";
 case Edition.99997_TEST_ONLY: return "EDITION_99997_TEST_ONLY";
@@ -1001,7 +1007,12 @@ readonly unverifiedLazy?: boolean | undefined,
  * is a formalization for deprecating fields.
  */
 readonly deprecated?: boolean | undefined,
-/** For Google-internal migration only. Do not use. */
+/**
+ * DEPRECATED. DO NOT USE!
+ * For Google-internal migration only. Do not use.
+ * 
+ * @deprecated
+ */
 readonly weak?: boolean | undefined,
 /**
  * Indicate that the field value should not be printed out when using debug

package/src/proto/google/protobuf/timestamp.ts

@@ -104,15 +104,16 @@ export const protobufPackage = "google.protobuf";
 export interface Timestamp {
   $type: "google.protobuf.Timestamp";
   /**
-   * Represents seconds of UTC time since Unix epoch
-   * 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to
-   * 9999-12-31T23:59:59Z inclusive.
+   * Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must
+   * be between -315576000000 and 315576000000 inclusive (which corresponds to
+   * 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z).
    */
   readonly seconds: bigint;
   /**
-   * Non-negative fractions of a second at nanosecond resolution. Negative
-   * second values with fractions must still have non-negative nanos values
-   * that count forward in time. Must be from 0 to 999,999,999
+   * Non-negative fractions of a second at nanosecond resolution. This field is
+   * the nanosecond portion of the duration, not an alternative to seconds.
+   * Negative second values with fractions must still have non-negative nanos
+   * values that count forward in time. Must be between 0 and 999,999,999
    * inclusive.
    */
   readonly nanos: number;
@@ -128,9 +129,7 @@ export const Timestamp: MessageFns<Timestamp, "google.protobuf.Timestamp"> = {
   encode(message: Timestamp, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
     if (message.seconds !== 0n) {
       if (BigInt.asIntN(64, message.seconds) !== message.seconds) {
-        throw new globalThis.Error(
-          "value provided for field message.seconds of type int64 too large",
-        );
+        throw new globalThis.Error("value provided for field message.seconds of type int64 too large");
       }
       writer.uint32(8).int64(message.seconds);
     }
@@ -206,24 +205,17 @@ messageTypeRegistry.set(Timestamp.$type, Timestamp);
 
 type Builtin = Date | Function | Uint8Array | string | number | boolean | bigint | undefined;
 
-export type DeepPartial<T> = T extends Builtin
-  ? T
-  : T extends globalThis.Array<infer U>
-    ? globalThis.Array<DeepPartial<U>>
-    : T extends ReadonlyArray<infer U>
-      ? ReadonlyArray<DeepPartial<U>>
-      : T extends { readonly $case: string }
-        ? { [K in keyof Omit<T, "$case">]?: DeepPartial<T[K]> } & { readonly $case: T["$case"] }
-        : T extends {}
-          ? { [K in Exclude<keyof T, "$type">]?: DeepPartial<T[K]> }
-          : Partial<T>;
+export type DeepPartial<T> = T extends Builtin ? T
+  : T extends globalThis.Array<infer U> ? globalThis.Array<DeepPartial<U>>
+  : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>>
+  : T extends { readonly $case: string }
+    ? { [K in keyof Omit<T, "$case">]?: DeepPartial<T[K]> } & { readonly $case: T["$case"] }
+  : T extends {} ? { [K in Exclude<keyof T, "$type">]?: DeepPartial<T[K]> }
+  : Partial<T>;
 
 type KeysOfUnion<T> = T extends T ? keyof T : never;
-export type Exact<P, I extends P> = P extends Builtin
-  ? P
-  : P & { [K in keyof P]: Exact<P[K], I[K]> } & {
-      [K in Exclude<keyof I, KeysOfUnion<P> | "$type">]: never;
-    };
+export type Exact<P, I extends P> = P extends Builtin ? P
+  : P & { [K in keyof P]: Exact<P[K], I[K]> } & { [K in Exclude<keyof I, KeysOfUnion<P> | "$type">]: never };
 
 function isSet(value: any): boolean {
   return value !== null && value !== undefined;

package/src/proto/typeRegistry.ts

@@ -20,14 +20,10 @@ export type UnknownMessage = { $type: string };
 export const messageTypeRegistry = new Map<string, MessageType>();
 
 type Builtin = Date | Function | Uint8Array | string | number | boolean | bigint | undefined;
-export type DeepPartial<T> = T extends Builtin
-  ? T
-  : T extends globalThis.Array<infer U>
-    ? globalThis.Array<DeepPartial<U>>
-    : T extends ReadonlyArray<infer U>
-      ? ReadonlyArray<DeepPartial<U>>
-      : T extends { readonly $case: string }
-        ? { [K in keyof Omit<T, "$case">]?: DeepPartial<T[K]> } & { readonly $case: T["$case"] }
-        : T extends {}
-          ? { [K in Exclude<keyof T, "$type">]?: DeepPartial<T[K]> }
-          : Partial<T>;
+export type DeepPartial<T> = T extends Builtin ? T
+  : T extends globalThis.Array<infer U> ? globalThis.Array<DeepPartial<U>>
+  : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>>
+  : T extends { readonly $case: string }
+    ? { [K in keyof Omit<T, "$case">]?: DeepPartial<T[K]> } & { readonly $case: T["$case"] }
+  : T extends {} ? { [K in Exclude<keyof T, "$type">]?: DeepPartial<T[K]> }
+  : Partial<T>;

package/src/record-batch-decoder.ts

@@ -49,6 +49,7 @@ import { CompressedVectorLoader, VectorLoader } from "apache-arrow/visitor/vecto
 import * as flatbuffers from "flatbuffers";
 
 import type { FlightData } from "./proto/Flight";
+import type { RecordBatchWithMetadata } from "./record-batch-with-metadata";
 
 const invalidMessageType = (type: MessageHeader) =>
   `Expected ${MessageHeader[type]} Message in stream, but was null or length 0.`;
@@ -240,7 +241,7 @@ abstract class RecordBatchReaderImpl<T extends TypeMap = any> implements RecordB
 
 export class RecordBatchStreamReaderFromFlightData<T extends TypeMap = any>
   extends RecordBatchReaderImpl<T>
-  implements AsyncIterableIterator<RecordBatch<T>>
+  implements AsyncIterableIterator<RecordBatchWithMetadata<T>>
 {
   protected _reader: AsyncIterator<FlightData>;
 
@@ -261,8 +262,8 @@ export class RecordBatchStreamReaderFromFlightData<T extends TypeMap = any>
   //   return true;
   // }
 
-  public [Symbol.asyncIterator](): AsyncIterableIterator<RecordBatch<T>> {
-    return this as AsyncIterableIterator<RecordBatch<T>>;
+  public [Symbol.asyncIterator](): AsyncIterableIterator<RecordBatchWithMetadata<T>> {
+    return this as AsyncIterableIterator<RecordBatchWithMetadata<T>>;
   }
 
   /*
@@ -320,7 +321,13 @@ export class RecordBatchStreamReaderFromFlightData<T extends TypeMap = any>
         this._recordBatchIndex++;
         const header = message.header();
         const recordBatch = this._loadRecordBatch(header, flight.dataBody);
-        return { done: false, value: recordBatch };
+        return {
+          done: false,
+          value: {
+            batch: recordBatch,
+            appMetadata: flight.appMetadata,
+          },
+        };
       } else if (message.isDictionaryBatch()) {
         this._dictionaryIndex++;
         const header = message.header();
@@ -333,7 +340,10 @@ export class RecordBatchStreamReaderFromFlightData<T extends TypeMap = any>
       this._recordBatchIndex++;
       return {
         done: false,
-        value: new _InternalEmptyPlaceholderRecordBatch<T>(this.schema),
+        value: {
+          batch: new _InternalEmptyPlaceholderRecordBatch<T>(this.schema),
+          appMetadata: new Uint8Array(0),
+        },
       };
     }
 

package/src/record-batch-with-metadata.ts

@@ -0,0 +1,6 @@
+import type { RecordBatch, TypeMap } from "apache-arrow";
+
+export type RecordBatchWithMetadata<T extends TypeMap = any> = {
+  readonly batch: RecordBatch<T>;
+  readonly appMetadata: Uint8Array;
+};

package/src/arrow-flight-sql.ts

@@ -1,98 +0,0 @@
-import type { CallOptions } from "nice-grpc";
-
-import type { ClientOptions, HostOrChannel, RemoveTypeUrl } from "./proto-utils";
-
-import { ArrowFlightClient } from "./arrow-flight";
-import { Any } from "./proto/any";
-import {
-  FlightDescriptor,
-  FlightDescriptor_DescriptorType,
-  type FlightInfo,
-  type FlightServiceDefinition,
-} from "./proto/Flight";
-import {
-  CommandGetCatalogs,
-  CommandGetDbSchemas,
-  CommandGetTables,
-  CommandGetTableTypes,
-  CommandStatementQuery,
-} from "./proto/FlightSql";
-
-export class ArrowFlightSqlClient {
-  private inner: ArrowFlightClient;
-
-  constructor(config: HostOrChannel, options: ClientOptions<FlightServiceDefinition> = {}) {
-    this.inner = new ArrowFlightClient(config, options);
-  }
-
-  executeFlightInfo(request: FlightInfo, options?: CallOptions) {
-    return this.inner.executeFlightInfo(request, options);
-  }
-
-  async getCatalogs(request: RemoveTypeUrl<CommandGetCatalogs>, options?: CallOptions) {
-    const descriptor = createCommandDescriptor(
-      CommandGetCatalogs.$type,
-      CommandGetCatalogs.encode({
-        $type: CommandGetCatalogs.$type,
-        ...request,
-      }).finish(),
-    );
-    return this.inner.getFlightInfo(descriptor, options);
-  }
-
-  async getDbSchemas(request: RemoveTypeUrl<CommandGetDbSchemas>, options?: CallOptions) {
-    const descriptor = createCommandDescriptor(
-      CommandGetDbSchemas.$type,
-      CommandGetDbSchemas.encode({
-        $type: CommandGetDbSchemas.$type,
-        ...request,
-      }).finish(),
-    );
-    return this.inner.getFlightInfo(descriptor, options);
-  }
-
-  async getTables(request: RemoveTypeUrl<CommandGetTables>, options?: CallOptions) {
-    const descriptor = createCommandDescriptor(
-      CommandGetTables.$type,
-      CommandGetTables.encode({
-        $type: CommandGetTables.$type,
-        ...request,
-      }).finish(),
-    );
-    return this.inner.getFlightInfo(descriptor, options);
-  }
-
-  async getTableTypes(request: RemoveTypeUrl<CommandGetTableTypes>, options?: CallOptions) {
-    const descriptor = createCommandDescriptor(
-      CommandGetTableTypes.$type,
-      CommandGetTableTypes.encode({
-        $type: CommandGetTableTypes.$type,
-        ...request,
-      }).finish(),
-    );
-    return this.inner.getFlightInfo(descriptor, options);
-  }
-
-  async executeQuery(request: RemoveTypeUrl<CommandStatementQuery>, options?: CallOptions) {
-    const descriptor = createCommandDescriptor(
-      CommandStatementQuery.$type,
-      CommandStatementQuery.encode({
-        $type: CommandStatementQuery.$type,
-        ...request,
-      }).finish(),
-    );
-    return this.inner.getFlightInfo(descriptor, options);
-  }
-}
-
-function createCommandDescriptor(typeUrl: string, value: Uint8Array): FlightDescriptor {
-  const cmd = Any.create({
-    typeUrl: `type.googleapis.com/${typeUrl}`,
-    value,
-  });
-
-  return FlightDescriptor.create({
-    type: FlightDescriptor_DescriptorType.CMD,
-    cmd: Any.encode(cmd).finish(),
-  });
-}

package/src/arrow-flight.ts

@@ -1,126 +0,0 @@
-import type { RecordBatch, Schema } from "apache-arrow";
-
-import { type CallOptions, createClient } from "nice-grpc";
-
-import { decodeFlightDataStream, decodeSchemaFromFlightInfo } from "./arrow-utils";
-import { type ClientOptions, createChannelFromConfig, type HostOrChannel } from "./proto-utils";
-import {
-  type FlightData,
-  type FlightDescriptor,
-  type FlightInfo,
-  type FlightServiceClient,
-  FlightServiceDefinition,
-  type HandshakeRequest,
-  type HandshakeResponse,
-  type PutResult,
-  type Ticket,
-} from "./proto/Flight";
-
-export class ArrowFlightClient {
-  private client: FlightServiceClient;
-
-  constructor(config: HostOrChannel, options: ClientOptions<FlightServiceDefinition> = {}) {
-    const channel = createChannelFromConfig(config);
-
-    this.client = createClient(FlightServiceDefinition, channel, options.defaultCallOptions);
-  }
-
-  executeFlightInfo(info: FlightInfo, options?: CallOptions): AsyncGenerator<RecordBatch> {
-    const schema = decodeSchemaFromFlightInfo(info);
-    if (!schema) {
-      throw new Error("FlightInfo must have a schema");
-    }
-
-    // oxlint-disable-next-line no-this-alias
-    const client = this;
-
-    return (async function* () {
-      for (const endpoint of info.endpoint) {
-        if (endpoint.ticket === undefined) {
-          continue;
-        }
-
-        yield* client.doGet(endpoint.ticket, { schema, ...options });
-      }
-    })();
-  }
-
-  /**
-   * Handshake between client and server.
-   *
-   * Depending on the server, the handshake may be required to determine the
-   * token that should be used for future operations. Both request and response
-   * are streams to allow multiple round-trips depending on auth mechanism.
-   */
-  handshake(
-    request: AsyncIterable<HandshakeRequest>,
-    options?: CallOptions,
-  ): AsyncIterable<HandshakeResponse> {
-    return this.client.handshake(request, options);
-  }
-
-  /** Get a list of available streams given a particular criteria. */
-  // listFlights(
-  //   request: proto.arrow_flight.Criteria,
-  //   options?: ClientCallOptions,
-  // ): AsyncIterable<proto.arrow_flight.FlightInfo>;
-
-  getFlightInfo(request: FlightDescriptor, options?: CallOptions): Promise<FlightInfo> {
-    return this.client.getFlightInfo(request, options);
-  }
-
-  // /** Start a query and get information to poll its execution status. */
-  // pollFlightInfo(
-  //   request: proto.arrow_flight.FlightDescriptor,
-  //   options?: ClientCallOptions,
-  // ): Promise<proto.arrow_flight.PollInfo>;
-
-  // /** Get the schema for a given FlightDescriptor. */
-  // getSchema(
-  //   request: proto.arrow_flight.FlightDescriptor,
-  //   options?: ClientCallOptions,
-  // ): Promise<proto.arrow_flight.SchemaResult>;
-
-  /**
-   * Retrieve a single stream associated with a particular descriptor
-   * associated with the referenced ticket. A Flight can be composed of one or
-   * more streams where each stream can be retrieved using a separate opaque
-   * ticket that the flight service uses for managing a collection of streams.
-   */
-  doGet(request: Ticket, options: { schema: Schema } & CallOptions): AsyncIterable<RecordBatch> {
-    const { schema: expectedSchema } = options;
-    return decodeFlightDataStream(this.client.doGet(request, options), {
-      expectedSchema,
-    });
-  }
-
-  /**
-   * Push a stream to the flight service associated with a particular
-   * flight stream. This allows a client of a flight service to upload a stream
-   * of data. Depending on the particular flight service, a client consumer
-   * could be allowed to upload a single stream per descriptor or an unlimited
-   * number. In the latter, the service might implement a 'seal' action that
-   * can be applied to a descriptor once all streams are uploaded.
-   */
-  doPut(request: AsyncIterable<FlightData>, options?: CallOptions): AsyncIterable<PutResult> {
-    return this.client.doPut(request, options);
-  }
-
-  // /** Open a bidirectional data channel for a given descriptor. */
-  // doExchange(
-  //   request: AsyncIterable<proto.arrow_flight.FlightData>,
-  //   options?: ClientCallOptions,
-  // ): AsyncIterable<proto.arrow_flight.FlightData>;
-
-  // /** Execute a specific action against the flight service. */
-  // doAction(
-  //   request: proto.arrow_flight.Action,
-  //   options?: ClientCallOptions,
-  // ): AsyncIterable<proto.arrow_flight.Result>;
-
-  // /** Get all available action types. */
-  // listActions(
-  //   request: proto.arrow_flight.Empty,
-  //   options?: ClientCallOptions,
-  // ): AsyncIterable<proto.arrow_flight.ActionType>;
-}