@dremio/js-sdk 0.1.2 → 0.3.0

package/dist/oss/jobs/Job.d.ts

@@ -19,13 +19,18 @@ export declare class Job {
     constructor(properties: JobProperties, config: SonarV3Config, observe: (id: string) => Observable<JobResult>);
     get settled(): boolean;
     get results(): {
-        jsonBatches: <T extends Record<string, unknown> = Record<string, unknown>>() => AsyncGenerator<{
+        jsonBatches: <T extends Record<string, unknown> = Record<string, unknown>>(args?: {
+            limit?: number;
+            offset?: number;
+        }) => AsyncGenerator<{
             readonly columns: Record<Extract<keyof T, string>, unknown[] | Float64Array<ArrayBuffer> | Float32Array<ArrayBuffer> | Int32Array<ArrayBuffer>>;
             readonly rows: T[];
             readonly schema: {
                 fields: import("./utils/JobResultsResponse.ts").JobResultsSchema<Extract<keyof T, string>>;
             };
+            readonly totalRows: number;
         }, void, unknown>;
+        slice: <T extends Record<string, unknown> = Record<string, unknown>>(start?: number, end?: number) => Promise<T[]>;
     };
     cancel(): Promise<Err<any> | Ok<undefined>>;
 }

package/dist/oss/jobs/Job.js

@@ -67,17 +67,31 @@ export class Job {
     get results() {
         return {
             jsonBatches: this.#jsonBatches.bind(this),
+            slice: this.#slice.bind(this),
         };
     }
-    async *#jsonBatches() {
+    async *#jsonBatches(args = {}) {
+        if (typeof args.limit === "number" && args.limit < 0) {
+            throw new Error("Limit cannot be negative");
+        }
+        if (typeof args.offset === "number" && args.offset < 0) {
+            throw new Error("Offset cannot be negative");
+        }
         // Wait for job to enter a settled state before attempting to fetch batches
         if (!this.settled) {
             await lastValueFrom(this.observer);
         }
-        const batch_size = 500;
+        const limitArg = args.limit ?? Infinity;
+        const offsetArg = args.offset || 0;
+        // Tracks whether there are more rows available from the job results endpoint
         let hasMore = true;
-        let offset = 0;
-        while (hasMore) {
+        // Tracks the currently requested offset. If the offset arg is provided, start from there instead of 0.
+        let offset = offsetArg;
+        // Keeps track of the total number of rows that need to be loaded
+        const stopAfterOffset = limitArg + offsetArg;
+        while (hasMore && offset < stopAfterOffset) {
+            // Make batch_size dynamic to allow for requesting a smaller final page
+            const batch_size = Math.min(MAX_BATCH_SIZE, stopAfterOffset - offset);
             const batch = await this.#config
                 .sonarV3Request(`job/${this.id}/results?offset=${offset}&limit=${batch_size}`)
                 .then((res) => res.json());
@@ -92,10 +106,18 @@ export class Job {
                     },
                     rows: batch.rows,
                     schema,
+                    totalRows: batch.rowCount,
                 };
             }
         }
     }
+    async #slice(start = 0, end = Infinity) {
+        const rows = [];
+        for await (const batch of this.#jsonBatches({ limit: end - start, offset: start })) {
+            rows.push(...batch.rows);
+        }
+        return rows;
+    }
     async cancel() {
         return this.#config
             .sonarV3Request(`job/${this.id}/cancel`, {
@@ -123,4 +145,5 @@ export const jobEntityToProperties = (id, entity) => ({
     startedAt: entity.startedAt ? new Date(entity.startedAt) : null,
     state: entity.jobState,
 });
+const MAX_BATCH_SIZE = 500;
 //# sourceMappingURL=Job.js.map

package/dist/oss/jobs/Job.js.map

@@ -1 +1 @@
-{"version":3,"file":"Job.js","sourceRoot":"","sources":["../../../src/oss/jobs/Job.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,EAAE,SAAS,EAAmB,MAAM,MAAM,CAAC;AAE1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,MAAM,OAAO,GAAG;IACL,YAAY,CAAgC;IAC5C,kBAAkB,CAAsC;IACxD,OAAO,CAA2B;IAClC,EAAE,CAAsB;IACxB,OAAO,CAA2B;IAClC,SAAS,CAA6B;IACtC,yBAAyB,CAA6C;IACtE,2BAA2B,CAA+C;IAC1E,QAAQ,CAA4B;IACpC,SAAS,CAA6B;IACtC,KAAK,CAAyB;IAEvB,QAAQ,CAAkB;IAC1C,OAAO,CAAgB;IAEvB,YACE,UAAyB,EACzB,MAAqB,EACrB,OAA8C;QAE9C,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;QAC5C,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QACtE,IAAI,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAC3B,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACjB,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAC1B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;oBACb,IAAI,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;wBACnB,MAAM,MAAM,CAAC,KAAK,CAAC;oBACrB,CAAC;oBACD,OAAO,MAAM,CAAC,KAAK,CAAC;gBACtB,CAAC,CAAC,CACH,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO,CACL,IAAI,CAAC,KAAK,KAAK,WAAW;YAC1B,IAAI,CAAC,KAAK,KAAK,QAAQ;YACvB,IAAI,CAAC,KAAK,KAAK,eAAe;YAC9B,IAAI,CAAC,KAAK,KAAK,UAAU,CAC1B,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,CAAC,YAAY;QACjB,2EAA2E;QAC3E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,UAAU,GAAG,GAAG,CAAC;QACvB,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,OAAO,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO;iBAC7B,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,mBAAmB,MAAM,UAAU,UAAU,EAAE,CAAC;iBAC7E,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAoC,CAAC,CAAC;YAE/D,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YAC5B,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM;oBACJ,IAAI,OAAO;wBACT,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,CAAC;oBACD,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM;iBACE,CAAC;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO;aAChB,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,SAAS,EAAE;YACvC,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,MAAM;SACf,CAAC;aACD,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;aACzB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,EAAU,EACV,MA6CC,EACD,EAAE,CACF,CAAC;IACC,YAAY,EAAE,MAAM,CAAC,YAAY;IACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;IAC7C,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;IACzD,EAAE;IACF,OAAO,EAAE,IAAsB;IAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;IAC3B,yBAAyB,EAAE,MAAM,CAAC,yBAAyB;QACzD,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC;QAC5C,CAAC,CAAC,IAAI;IACR,2BAA2B,EAAE,MAAM,CAAC,2BAA2B;QAC7D,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC;QAC9C,CAAC,CAAC,IAAI;IACR,QAAQ,EAAE,MAAM,CAAC,QAAQ;IACzB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;IAC/D,KAAK,EAAE,MAAM,CAAC,QAAQ;CACvB,CAAU,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { SonarV3Config } from \"../../common/Config.ts\";\nimport type { Problem } from \"../../common/Problem.ts\";\nimport { Err, Ok, Result } from \"ts-results-es\";\nimport { lastValueFrom, map, of, switchMap, type Observable } from \"rxjs\";\nimport type { JobResultsResponse } from \"./utils/JobResultsResponse.ts\";\nimport { mapRowsToColumns } from \"./utils/mapRowsToColumns.ts\";\nimport { mapRowData } from \"./utils/mapRowData.ts\";\n\nexport class Job {\n  readonly acceleration: JobProperties[\"acceleration\"];\n  readonly cancellationReason: JobProperties[\"cancellationReason\"];\n  readonly endedAt: JobProperties[\"endedAt\"];\n  readonly id: JobProperties[\"id\"];\n  readonly problem: JobProperties[\"problem\"];\n  readonly queryType: JobProperties[\"queryType\"];\n  readonly resourceSchedulingEndedAt: JobProperties[\"resourceSchedulingEndedAt\"];\n  readonly resourceSchedulingStartedAt: JobProperties[\"resourceSchedulingStartedAt\"];\n  readonly rowCount: JobProperties[\"rowCount\"];\n  readonly startedAt: JobProperties[\"startedAt\"];\n  readonly state: JobProperties[\"state\"];\n\n  public readonly observer: Observable<Job>;\n  #config: SonarV3Config;\n\n  constructor(\n    properties: JobProperties,\n    config: SonarV3Config,\n    observe: (id: string) => Observable<JobResult>,\n  ) {\n    this.acceleration = properties.acceleration;\n    this.cancellationReason = properties.cancellationReason;\n    this.endedAt = properties.endedAt;\n    this.id = properties.id;\n    this.problem = properties.problem;\n    this.queryType = properties.queryType;\n    this.resourceSchedulingEndedAt = properties.resourceSchedulingEndedAt;\n    this.resourceSchedulingStartedAt = properties.resourceSchedulingStartedAt;\n    this.rowCount = properties.rowCount;\n    this.startedAt = properties.startedAt;\n    this.state = properties.state;\n    this.#config = config;\n    this.observer = of(this).pipe(\n      switchMap((job) => {\n        if (!job.settled) {\n          return observe(this.id).pipe(\n            map((result) => {\n              if (result.isErr()) {\n                throw result.error;\n              }\n              return result.value;\n            }),\n          );\n        } else {\n          return of(job);\n        }\n      }),\n    );\n  }\n\n  get settled() {\n    return (\n      this.state === \"COMPLETED\" ||\n      this.state === \"FAILED\" ||\n      this.state === \"INVALID_STATE\" ||\n      this.state === \"CANCELED\"\n    );\n  }\n\n  get results() {\n    return {\n      jsonBatches: this.#jsonBatches.bind(this),\n    };\n  }\n\n  async *#jsonBatches<T extends Record<string, unknown> = Record<string, unknown>>() {\n    // Wait for job to enter a settled state before attempting to fetch batches\n    if (!this.settled) {\n      await lastValueFrom(this.observer);\n    }\n\n    const batch_size = 500;\n    let hasMore = true;\n    let offset = 0;\n    while (hasMore) {\n      const batch = await this.#config\n        .sonarV3Request(`job/${this.id}/results?offset=${offset}&limit=${batch_size}`)\n        .then((res) => res.json() as Promise<JobResultsResponse<T>>);\n\n      offset += batch.rows.length;\n      hasMore = batch.rows.length === batch_size;\n\n      if (batch.rows.length) {\n        const schema = { fields: batch.schema };\n        mapRowData(batch.rows, batch.schema);\n        yield {\n          get columns() {\n            return mapRowsToColumns(batch);\n          },\n          rows: batch.rows,\n          schema,\n        } as const;\n      }\n    }\n  }\n\n  async cancel() {\n    return this.#config\n      .sonarV3Request(`job/${this.id}/cancel`, {\n        keepalive: true,\n        method: \"POST\",\n      })\n      .then(() => Ok(undefined))\n      .catch((e) => Err(e));\n  }\n}\n\nexport const jobEntityToProperties = (\n  id: string,\n  entity: {\n    jobState:\n      | \"NOT_SUBMITTED\"\n      | \"STARTING\"\n      | \"RUNNING\"\n      | \"COMPLETED\"\n      | \"CANCELED\"\n      | \"FAILED\"\n      | \"CANCELLATION_REQUESTED\"\n      | \"PLANNING\"\n      | \"PENDING\"\n      | \"METADATA_RETRIEVAL\"\n      | \"QUEUED\"\n      | \"ENGINE_START\"\n      | \"EXECUTION_PLANNING\"\n      | \"INVALID_STATE\";\n    rowCount: number | null;\n    errorMessage: string;\n    startedAt: string | null;\n    endedAt: string | null;\n    acceleration: {\n      reflectionRelationships: {\n        datasetId: string;\n        reflectionId: string;\n        relationship: string;\n      }[];\n    };\n    queryType:\n      | \"UI_RUN\"\n      | \"UI_PREVIEW\"\n      | \"UI_INTERNAL_PREVIEW\"\n      | \"UI_INTERNAL_RUN\"\n      | \"UI_EXPORT\"\n      | \"ODBC\"\n      | \"JDBC\"\n      | \"REST\"\n      | \"ACCELERATOR_CREATE\"\n      | \"ACCELERATOR_DROP\"\n      | \"UNKNOWN\"\n      | \"PREPARE_INTERNAL\"\n      | \"ACCELERATOR_EXPLAIN\"\n      | \"UI_INITIAL_PREVIEW\";\n    resourceSchedulingStartedAt: string | null;\n    resourceSchedulingEndedAt: string | null;\n    cancellationReason: string | null;\n  },\n) =>\n  ({\n    acceleration: entity.acceleration,\n    cancellationReason: entity.cancellationReason,\n    endedAt: entity.endedAt ? new Date(entity.endedAt) : null,\n    id,\n    problem: null as Problem | null,\n    queryType: entity.queryType,\n    resourceSchedulingEndedAt: entity.resourceSchedulingEndedAt\n      ? new Date(entity.resourceSchedulingEndedAt)\n      : null,\n    resourceSchedulingStartedAt: entity.resourceSchedulingStartedAt\n      ? new Date(entity.resourceSchedulingStartedAt)\n      : null,\n    rowCount: entity.rowCount,\n    startedAt: entity.startedAt ? new Date(entity.startedAt) : null,\n    state: entity.jobState,\n  }) as const;\n\ntype JobProperties = ReturnType<typeof jobEntityToProperties>;\n\nexport type JobResult = Result<Job, unknown>;\n"]}
+{"version":3,"file":"Job.js","sourceRoot":"","sources":["../../../src/oss/jobs/Job.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,EAAE,SAAS,EAAmB,MAAM,MAAM,CAAC;AAE1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEnD,MAAM,OAAO,GAAG;IACL,YAAY,CAAgC;IAC5C,kBAAkB,CAAsC;IACxD,OAAO,CAA2B;IAClC,EAAE,CAAsB;IACxB,OAAO,CAA2B;IAClC,SAAS,CAA6B;IACtC,yBAAyB,CAA6C;IACtE,2BAA2B,CAA+C;IAC1E,QAAQ,CAA4B;IACpC,SAAS,CAA6B;IACtC,KAAK,CAAyB;IAEvB,QAAQ,CAAkB;IAC1C,OAAO,CAAgB;IAEvB,YACE,UAAyB,EACzB,MAAqB,EACrB,OAA8C;QAE9C,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;QAC5C,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC,kBAAkB,CAAC;QACxD,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,yBAAyB,GAAG,UAAU,CAAC,yBAAyB,CAAC;QACtE,IAAI,CAAC,2BAA2B,GAAG,UAAU,CAAC,2BAA2B,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAC3B,SAAS,CAAC,CAAC,GAAG,EAAE,EAAE;YAChB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACjB,OAAO,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAC1B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;oBACb,IAAI,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC;wBACnB,MAAM,MAAM,CAAC,KAAK,CAAC;oBACrB,CAAC;oBACD,OAAO,MAAM,CAAC,KAAK,CAAC;gBACtB,CAAC,CAAC,CACH,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;YACjB,CAAC;QACH,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO,CACL,IAAI,CAAC,KAAK,KAAK,WAAW;YAC1B,IAAI,CAAC,KAAK,KAAK,QAAQ;YACvB,IAAI,CAAC,KAAK,KAAK,eAAe;YAC9B,IAAI,CAAC,KAAK,KAAK,UAAU,CAC1B,CAAC;IACJ,CAAC;IAED,IAAI,OAAO;QACT,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YACzC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,CAAC,YAAY,CACjB,OAA4C,EAAE;QAE9C,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,2EAA2E;QAC3E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QAEnC,6EAA6E;QAC7E,IAAI,OAAO,GAAG,IAAI,CAAC;QAEnB,uGAAuG;QACvG,IAAI,MAAM,GAAG,SAAS,CAAC;QAEvB,iEAAiE;QACjE,MAAM,eAAe,GAAG,QAAQ,GAAG,SAAS,CAAC;QAE7C,OAAO,OAAO,IAAI,MAAM,GAAG,eAAe,EAAE,CAAC;YAC3C,uEAAuE;YACvE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,GAAG,MAAM,CAAC,CAAC;YAEtE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO;iBAC7B,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,mBAAmB,MAAM,UAAU,UAAU,EAAE,CAAC;iBAC7E,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAoC,CAAC,CAAC;YAE/D,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YAC5B,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;gBACxC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrC,MAAM;oBACJ,IAAI,OAAO;wBACT,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBACjC,CAAC;oBACD,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM;oBACN,SAAS,EAAE,KAAK,CAAC,QAAQ;iBACjB,CAAC;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CACV,QAAgB,CAAC,EACjB,MAAc,QAAQ;QAEtB,MAAM,IAAI,GAAQ,EAAE,CAAC;QACrB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,CAAC,YAAY,CAAI,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACtF,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,OAAO;aAChB,cAAc,CAAC,OAAO,IAAI,CAAC,EAAE,SAAS,EAAE;YACvC,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,MAAM;SACf,CAAC;aACD,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;aACzB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,EAAU,EACV,MA6CC,EACD,EAAE,CACF,CAAC;IACC,YAAY,EAAE,MAAM,CAAC,YAAY;IACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;IAC7C,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;IACzD,EAAE;IACF,OAAO,EAAE,IAAsB;IAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;IAC3B,yBAAyB,EAAE,MAAM,CAAC,yBAAyB;QACzD,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,yBAAyB,CAAC;QAC5C,CAAC,CAAC,IAAI;IACR,2BAA2B,EAAE,MAAM,CAAC,2BAA2B;QAC7D,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,2BAA2B,CAAC;QAC9C,CAAC,CAAC,IAAI;IACR,QAAQ,EAAE,MAAM,CAAC,QAAQ;IACzB,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;IAC/D,KAAK,EAAE,MAAM,CAAC,QAAQ;CACvB,CAAU,CAAC;AAMd,MAAM,cAAc,GAAG,GAAG,CAAC","sourcesContent":["/*\n * Copyright (C) 2024-2025 Dremio Corporation\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport type { SonarV3Config } from \"../../common/Config.ts\";\nimport type { Problem } from \"../../common/Problem.ts\";\nimport { Err, Ok, Result } from \"ts-results-es\";\nimport { lastValueFrom, map, of, switchMap, type Observable } from \"rxjs\";\nimport type { JobResultsResponse } from \"./utils/JobResultsResponse.ts\";\nimport { mapRowsToColumns } from \"./utils/mapRowsToColumns.ts\";\nimport { mapRowData } from \"./utils/mapRowData.ts\";\n\nexport class Job {\n  readonly acceleration: JobProperties[\"acceleration\"];\n  readonly cancellationReason: JobProperties[\"cancellationReason\"];\n  readonly endedAt: JobProperties[\"endedAt\"];\n  readonly id: JobProperties[\"id\"];\n  readonly problem: JobProperties[\"problem\"];\n  readonly queryType: JobProperties[\"queryType\"];\n  readonly resourceSchedulingEndedAt: JobProperties[\"resourceSchedulingEndedAt\"];\n  readonly resourceSchedulingStartedAt: JobProperties[\"resourceSchedulingStartedAt\"];\n  readonly rowCount: JobProperties[\"rowCount\"];\n  readonly startedAt: JobProperties[\"startedAt\"];\n  readonly state: JobProperties[\"state\"];\n\n  public readonly observer: Observable<Job>;\n  #config: SonarV3Config;\n\n  constructor(\n    properties: JobProperties,\n    config: SonarV3Config,\n    observe: (id: string) => Observable<JobResult>,\n  ) {\n    this.acceleration = properties.acceleration;\n    this.cancellationReason = properties.cancellationReason;\n    this.endedAt = properties.endedAt;\n    this.id = properties.id;\n    this.problem = properties.problem;\n    this.queryType = properties.queryType;\n    this.resourceSchedulingEndedAt = properties.resourceSchedulingEndedAt;\n    this.resourceSchedulingStartedAt = properties.resourceSchedulingStartedAt;\n    this.rowCount = properties.rowCount;\n    this.startedAt = properties.startedAt;\n    this.state = properties.state;\n    this.#config = config;\n    this.observer = of(this).pipe(\n      switchMap((job) => {\n        if (!job.settled) {\n          return observe(this.id).pipe(\n            map((result) => {\n              if (result.isErr()) {\n                throw result.error;\n              }\n              return result.value;\n            }),\n          );\n        } else {\n          return of(job);\n        }\n      }),\n    );\n  }\n\n  get settled() {\n    return (\n      this.state === \"COMPLETED\" ||\n      this.state === \"FAILED\" ||\n      this.state === \"INVALID_STATE\" ||\n      this.state === \"CANCELED\"\n    );\n  }\n\n  get results() {\n    return {\n      jsonBatches: this.#jsonBatches.bind(this),\n      slice: this.#slice.bind(this),\n    };\n  }\n\n  async *#jsonBatches<T extends Record<string, unknown> = Record<string, unknown>>(\n    args: { limit?: number; offset?: number } = {},\n  ) {\n    if (typeof args.limit === \"number\" && args.limit < 0) {\n      throw new Error(\"Limit cannot be negative\");\n    }\n\n    if (typeof args.offset === \"number\" && args.offset < 0) {\n      throw new Error(\"Offset cannot be negative\");\n    }\n\n    // Wait for job to enter a settled state before attempting to fetch batches\n    if (!this.settled) {\n      await lastValueFrom(this.observer);\n    }\n\n    const limitArg = args.limit ?? Infinity;\n    const offsetArg = args.offset || 0;\n\n    // Tracks whether there are more rows available from the job results endpoint\n    let hasMore = true;\n\n    // Tracks the currently requested offset. If the offset arg is provided, start from there instead of 0.\n    let offset = offsetArg;\n\n    // Keeps track of the total number of rows that need to be loaded\n    const stopAfterOffset = limitArg + offsetArg;\n\n    while (hasMore && offset < stopAfterOffset) {\n      // Make batch_size dynamic to allow for requesting a smaller final page\n      const batch_size = Math.min(MAX_BATCH_SIZE, stopAfterOffset - offset);\n\n      const batch = await this.#config\n        .sonarV3Request(`job/${this.id}/results?offset=${offset}&limit=${batch_size}`)\n        .then((res) => res.json() as Promise<JobResultsResponse<T>>);\n\n      offset += batch.rows.length;\n      hasMore = batch.rows.length === batch_size;\n\n      if (batch.rows.length) {\n        const schema = { fields: batch.schema };\n        mapRowData(batch.rows, batch.schema);\n        yield {\n          get columns() {\n            return mapRowsToColumns(batch);\n          },\n          rows: batch.rows,\n          schema,\n          totalRows: batch.rowCount,\n        } as const;\n      }\n    }\n  }\n\n  async #slice<T extends Record<string, unknown> = Record<string, unknown>>(\n    start: number = 0,\n    end: number = Infinity,\n  ) {\n    const rows: T[] = [];\n    for await (const batch of this.#jsonBatches<T>({ limit: end - start, offset: start })) {\n      rows.push(...batch.rows);\n    }\n    return rows;\n  }\n\n  async cancel() {\n    return this.#config\n      .sonarV3Request(`job/${this.id}/cancel`, {\n        keepalive: true,\n        method: \"POST\",\n      })\n      .then(() => Ok(undefined))\n      .catch((e) => Err(e));\n  }\n}\n\nexport const jobEntityToProperties = (\n  id: string,\n  entity: {\n    jobState:\n      | \"NOT_SUBMITTED\"\n      | \"STARTING\"\n      | \"RUNNING\"\n      | \"COMPLETED\"\n      | \"CANCELED\"\n      | \"FAILED\"\n      | \"CANCELLATION_REQUESTED\"\n      | \"PLANNING\"\n      | \"PENDING\"\n      | \"METADATA_RETRIEVAL\"\n      | \"QUEUED\"\n      | \"ENGINE_START\"\n      | \"EXECUTION_PLANNING\"\n      | \"INVALID_STATE\";\n    rowCount: number | null;\n    errorMessage: string;\n    startedAt: string | null;\n    endedAt: string | null;\n    acceleration: {\n      reflectionRelationships: {\n        datasetId: string;\n        reflectionId: string;\n        relationship: string;\n      }[];\n    };\n    queryType:\n      | \"UI_RUN\"\n      | \"UI_PREVIEW\"\n      | \"UI_INTERNAL_PREVIEW\"\n      | \"UI_INTERNAL_RUN\"\n      | \"UI_EXPORT\"\n      | \"ODBC\"\n      | \"JDBC\"\n      | \"REST\"\n      | \"ACCELERATOR_CREATE\"\n      | \"ACCELERATOR_DROP\"\n      | \"UNKNOWN\"\n      | \"PREPARE_INTERNAL\"\n      | \"ACCELERATOR_EXPLAIN\"\n      | \"UI_INITIAL_PREVIEW\";\n    resourceSchedulingStartedAt: string | null;\n    resourceSchedulingEndedAt: string | null;\n    cancellationReason: string | null;\n  },\n) =>\n  ({\n    acceleration: entity.acceleration,\n    cancellationReason: entity.cancellationReason,\n    endedAt: entity.endedAt ? new Date(entity.endedAt) : null,\n    id,\n    problem: null as Problem | null,\n    queryType: entity.queryType,\n    resourceSchedulingEndedAt: entity.resourceSchedulingEndedAt\n      ? new Date(entity.resourceSchedulingEndedAt)\n      : null,\n    resourceSchedulingStartedAt: entity.resourceSchedulingStartedAt\n      ? new Date(entity.resourceSchedulingStartedAt)\n      : null,\n    rowCount: entity.rowCount,\n    startedAt: entity.startedAt ? new Date(entity.startedAt) : null,\n    state: entity.jobState,\n  }) as const;\n\ntype JobProperties = ReturnType<typeof jobEntityToProperties>;\n\nexport type JobResult = Result<Job, unknown>;\n\nconst MAX_BATCH_SIZE = 500;\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dremio/js-sdk",
-  "version": "0.1.2",
+  "version": "0.3.0",
   "description": "JavaScript library for the Dremio API",
   "keywords": [
     "dremio",