deepline 0.1.319 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/play.ts +8 -1
- package/dist/bundling-sources/sdk/src/release.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +20 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +11 -0
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +131 -3
- package/dist/cli/index.js +6 -3
- package/dist/cli/index.mjs +6 -3
- package/dist/index.d.mts +7 -2
- package/dist/index.d.ts +7 -2
- package/dist/index.js +100 -3
- package/dist/index.mjs +100 -3
- package/package.json +1 -1
|
@@ -1862,7 +1862,9 @@ function attachSdkQueryResultDatasetResult(
|
|
|
1862
1862
|
}
|
|
1863
1863
|
const datasetLimit =
|
|
1864
1864
|
finitePositiveInteger(dataset.returned_limit) ?? totalRows;
|
|
1865
|
-
const
|
|
1865
|
+
const effectiveCount = Math.min(totalRows, datasetLimit);
|
|
1866
|
+
const rawRows = rowsFromUnknown(raw?.rows);
|
|
1867
|
+
const previewRows = rawRows.slice(0, 25);
|
|
1866
1868
|
const fetchPage = async (
|
|
1867
1869
|
offset: number,
|
|
1868
1870
|
limit: number,
|
|
@@ -1927,10 +1929,15 @@ function attachSdkQueryResultDatasetResult(
|
|
|
1927
1929
|
datasetId: `sdk-tool-list:${toolId}:${stableHash(datasetScope)}:${datasetLimit}:${safeNonce}`,
|
|
1928
1930
|
count: Math.min(totalRows, datasetLimit),
|
|
1929
1931
|
previewRows,
|
|
1932
|
+
residentRows:
|
|
1933
|
+
rawRows.length >= effectiveCount
|
|
1934
|
+
? rawRows.slice(0, effectiveCount)
|
|
1935
|
+
: null,
|
|
1930
1936
|
sourceLabel: 'query result rows',
|
|
1931
1937
|
tableNamespace: null,
|
|
1932
1938
|
resolvers: {
|
|
1933
1939
|
count: async () => Math.min(totalRows, datasetLimit),
|
|
1940
|
+
at: async (index) => (await fetchPage(index, 1))[0],
|
|
1934
1941
|
peek: async (limit) => collectRows(limit),
|
|
1935
1942
|
materialize: async (limit) => collectRows(limit),
|
|
1936
1943
|
iterate: () =>
|
|
@@ -157,7 +157,10 @@ export const SDK_RELEASE = {
|
|
|
157
157
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
158
158
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
159
159
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
160
|
-
|
|
160
|
+
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
161
|
+
// exposed storage-dependent synchronous access. This deliberate minor
|
|
162
|
+
// release keeps lazy paging semantics independent of row residency.
|
|
163
|
+
version: '0.2.0',
|
|
161
164
|
contracts: {
|
|
162
165
|
api: {
|
|
163
166
|
name: 'sdk-http-api',
|
|
@@ -3658,6 +3658,7 @@ export class PlayContextImpl {
|
|
|
3658
3658
|
const originalRequestInput = requestInput as Record<string, unknown>;
|
|
3659
3659
|
const datasetLimit =
|
|
3660
3660
|
finitePositiveInteger(dataset.returned_limit) ?? totalRows;
|
|
3661
|
+
const effectiveCount = Math.min(totalRows, datasetLimit);
|
|
3661
3662
|
const previewRows = rows.slice(0, Math.min(rows.length, 25));
|
|
3662
3663
|
const executionNonce =
|
|
3663
3664
|
typeof wrapped.job_id === 'string' && wrapped.job_id.trim()
|
|
@@ -3718,12 +3719,15 @@ export class PlayContextImpl {
|
|
|
3718
3719
|
const playDataset = createDeferredPlayDataset({
|
|
3719
3720
|
datasetKind: 'csv',
|
|
3720
3721
|
datasetId,
|
|
3721
|
-
count:
|
|
3722
|
+
count: effectiveCount,
|
|
3722
3723
|
previewRows,
|
|
3724
|
+
residentRows:
|
|
3725
|
+
rows.length >= effectiveCount ? rows.slice(0, effectiveCount) : null,
|
|
3723
3726
|
sourceLabel: 'query result rows',
|
|
3724
3727
|
tableNamespace: null,
|
|
3725
3728
|
resolvers: {
|
|
3726
3729
|
count: async () => Math.min(totalRows, datasetLimit),
|
|
3730
|
+
at: async (index) => (await fetchPage(index, 1))[0],
|
|
3727
3731
|
peek: async (limit) => collectRows(limit),
|
|
3728
3732
|
materialize: async (limit) => collectRows(limit),
|
|
3729
3733
|
iterate: () =>
|
|
@@ -4803,6 +4807,10 @@ export class PlayContextImpl {
|
|
|
4803
4807
|
},
|
|
4804
4808
|
},
|
|
4805
4809
|
previewRows,
|
|
4810
|
+
residentRows:
|
|
4811
|
+
immediateMaterializedRows.length === successfulCount
|
|
4812
|
+
? immediateMaterializedRows
|
|
4813
|
+
: null,
|
|
4806
4814
|
tableNamespace: resolvedTableNamespace,
|
|
4807
4815
|
workProgress: {
|
|
4808
4816
|
total: totalInputCount,
|
|
@@ -5417,6 +5425,7 @@ export class PlayContextImpl {
|
|
|
5417
5425
|
previewRows: results
|
|
5418
5426
|
.slice(0, 5)
|
|
5419
5427
|
.map((row) => this.toMaterializedOutputRow(row)),
|
|
5428
|
+
residentRows: terminalRows,
|
|
5420
5429
|
tableNamespace: resolvedTableNamespace,
|
|
5421
5430
|
workProgress: {
|
|
5422
5431
|
total: totalInputCount,
|
|
@@ -8139,7 +8139,8 @@ export function createRuntimeBackedPlayDataset(input: {
|
|
|
8139
8139
|
}): PlayDataset<Record<string, unknown>> {
|
|
8140
8140
|
const csvRows =
|
|
8141
8141
|
input.datasetKind === 'csv' &&
|
|
8142
|
-
|
|
8142
|
+
typeof input.initialCount === 'number' &&
|
|
8143
|
+
(input.initialPreviewRows?.length ?? 0) >= input.initialCount
|
|
8143
8144
|
? [...(input.initialPreviewRows ?? [])]
|
|
8144
8145
|
: null;
|
|
8145
8146
|
if (csvRows) {
|
|
@@ -8151,6 +8152,7 @@ export function createRuntimeBackedPlayDataset(input: {
|
|
|
8151
8152
|
),
|
|
8152
8153
|
count: input.initialCount ?? csvRows.length,
|
|
8153
8154
|
previewRows: csvRows.slice(0, 10),
|
|
8155
|
+
residentRows: csvRows,
|
|
8154
8156
|
sourceLabel: input.sourceLabel ?? null,
|
|
8155
8157
|
resolvers: {
|
|
8156
8158
|
count: async () => csvRows.length,
|
|
@@ -8181,6 +8183,7 @@ export function createRuntimeBackedPlayDataset(input: {
|
|
|
8181
8183
|
input.tableNamespace,
|
|
8182
8184
|
),
|
|
8183
8185
|
count: input.initialCount ?? 0,
|
|
8186
|
+
knownCount: input.initialCount ?? null,
|
|
8184
8187
|
backing: {
|
|
8185
8188
|
storage: 'neon_sheet',
|
|
8186
8189
|
sheet: {
|
|
@@ -8222,6 +8225,22 @@ export function createRuntimeBackedPlayDataset(input: {
|
|
|
8222
8225
|
);
|
|
8223
8226
|
return rows.map((row) => row.data);
|
|
8224
8227
|
},
|
|
8228
|
+
at: async (index) => {
|
|
8229
|
+
const session = await getRuntimeDbSession(runtimeContext, {
|
|
8230
|
+
tableNamespace: input.tableNamespace,
|
|
8231
|
+
logicalTable: 'sheet_rows',
|
|
8232
|
+
operations: ['rows.read'],
|
|
8233
|
+
});
|
|
8234
|
+
const rows = await readRuntimeRows(
|
|
8235
|
+
requireRuntimePostgresSession(session),
|
|
8236
|
+
{
|
|
8237
|
+
limit: 1,
|
|
8238
|
+
offset: index,
|
|
8239
|
+
runId: input.runId ?? null,
|
|
8240
|
+
},
|
|
8241
|
+
);
|
|
8242
|
+
return rows[0]?.data;
|
|
8243
|
+
},
|
|
8225
8244
|
materialize: async (limit) => {
|
|
8226
8245
|
const pageSize = 1000;
|
|
8227
8246
|
const materialized: Record<string, unknown>[] = [];
|
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
createDeferredPlayDataset,
|
|
43
43
|
createPlayDataset,
|
|
44
44
|
isSerializedPlayDataset,
|
|
45
|
+
residentPlayDatasetRows,
|
|
45
46
|
trimSerializedPlayDatasetPreview,
|
|
46
47
|
type PlayDataset,
|
|
47
48
|
type SerializedPlayDataset,
|
|
@@ -1395,6 +1396,15 @@ function serializedListRowsFromResult(input: {
|
|
|
1395
1396
|
]);
|
|
1396
1397
|
const entries = [...names].flatMap((name) => {
|
|
1397
1398
|
const dataset = input.listDatasets?.[name];
|
|
1399
|
+
const liveDataset = input.value.extractedLists[name]?.get();
|
|
1400
|
+
const residentRows = liveDataset
|
|
1401
|
+
? residentPlayDatasetRows(liveDataset)
|
|
1402
|
+
: null;
|
|
1403
|
+
if (residentRows && (!dataset || residentRows.length >= dataset.count)) {
|
|
1404
|
+
return [
|
|
1405
|
+
[name, [...residentRows] as Array<Record<string, unknown>>] as const,
|
|
1406
|
+
];
|
|
1407
|
+
}
|
|
1398
1408
|
const preservedRows = preserved?.[name];
|
|
1399
1409
|
if (preservedRows && (!dataset || preservedRows.length >= dataset.count)) {
|
|
1400
1410
|
return [[name, preservedRows]] as const;
|
|
@@ -1509,6 +1519,7 @@ function createDatasetFromSerializedToolList(
|
|
|
1509
1519
|
count: serialized.count,
|
|
1510
1520
|
backing: serialized.backing,
|
|
1511
1521
|
previewRows: preview,
|
|
1522
|
+
residentRows: isPartialPreview ? null : sourceRows,
|
|
1512
1523
|
sourceLabel: serialized.sourceLabel ?? null,
|
|
1513
1524
|
tableNamespace: serialized.tableNamespace ?? null,
|
|
1514
1525
|
workProgress: serialized._metadata?.workProgress,
|
|
@@ -2,6 +2,7 @@ import type { PlayExecutionFileRef } from './file-refs';
|
|
|
2
2
|
|
|
3
3
|
const PLAY_DATASET_BRAND = Symbol.for('deepline.play.dataset');
|
|
4
4
|
const NODE_INSPECT_CUSTOM = Symbol.for('nodejs.util.inspect.custom');
|
|
5
|
+
const residentRowsByDataset = new WeakMap<object, readonly unknown[]>();
|
|
5
6
|
const DEFAULT_MATERIALIZE_LIMIT = 10_000;
|
|
6
7
|
export const PLAY_DATASET_EXECUTION_PAGE_ROWS = 1_000;
|
|
7
8
|
export const PLAY_DATASET_EXECUTION_PAGE_BYTES = 64 * 1024 * 1024;
|
|
@@ -104,8 +105,9 @@ export type PlayDatasetTransformOptions = {
|
|
|
104
105
|
* runtime control. Use `count()` and `peek()` for bounded inspection. Use
|
|
105
106
|
* `materialize(limit)` or async iteration only when the dataset is intentionally
|
|
106
107
|
* small and bounded. `PlayDataset` intentionally does not expose `.rows`,
|
|
107
|
-
* `.toArray()`,
|
|
108
|
-
* persisted rows into memory
|
|
108
|
+
* `.toArray()`, `.length`, numeric indexing, spread, or synchronous iteration;
|
|
109
|
+
* those hide the runtime cost of loading persisted rows into memory or make
|
|
110
|
+
* behavior depend on whether rows happen to be resident.
|
|
109
111
|
*
|
|
110
112
|
* @sdkReference runtime 190
|
|
111
113
|
*/
|
|
@@ -125,6 +127,10 @@ export interface PlayDataset<T> extends AsyncIterable<T> {
|
|
|
125
127
|
count(): Promise<number>;
|
|
126
128
|
/** Preview rows. */
|
|
127
129
|
peek(limit?: number): Promise<T[]>;
|
|
130
|
+
/** First row, loading it asynchronously when necessary. */
|
|
131
|
+
first(): Promise<T | undefined>;
|
|
132
|
+
/** Row at an array-style index, loading it asynchronously when necessary. */
|
|
133
|
+
at(index: number): Promise<T | undefined>;
|
|
128
134
|
map<U>(
|
|
129
135
|
mapper: (row: T, index: number) => U | Promise<U>,
|
|
130
136
|
options?: PlayDatasetTransformOptions,
|
|
@@ -164,6 +170,7 @@ export interface PlayDataset<T> extends AsyncIterable<T> {
|
|
|
164
170
|
type PlayDatasetResolvers<T> = {
|
|
165
171
|
count: () => Promise<number>;
|
|
166
172
|
peek: (limit: number) => Promise<T[]>;
|
|
173
|
+
at?: (index: number) => Promise<T | undefined>;
|
|
167
174
|
materialize: (limit?: number) => Promise<T[]>;
|
|
168
175
|
materializeFullPersistedDataset?: (limit?: number) => Promise<T[]>;
|
|
169
176
|
iterate: () => AsyncIterable<T>;
|
|
@@ -223,6 +230,13 @@ export function isPlayDataset<T>(value: unknown): value is PlayDataset<T> {
|
|
|
223
230
|
);
|
|
224
231
|
}
|
|
225
232
|
|
|
233
|
+
/** Internal compatibility view for preserving proven-complete rows on replay. */
|
|
234
|
+
export function residentPlayDatasetRows<T>(
|
|
235
|
+
dataset: PlayDataset<T>,
|
|
236
|
+
): readonly T[] | null {
|
|
237
|
+
return (residentRowsByDataset.get(dataset as object) as readonly T[]) ?? null;
|
|
238
|
+
}
|
|
239
|
+
|
|
226
240
|
export function isSerializedPlayDataset<T>(
|
|
227
241
|
value: unknown,
|
|
228
242
|
): value is SerializedPlayDataset<T> {
|
|
@@ -317,6 +331,7 @@ export function deserializePlayDatasetCell<T>(
|
|
|
317
331
|
count: rows.length,
|
|
318
332
|
backing: dataset.backing,
|
|
319
333
|
previewRows: rows.slice(0, 10),
|
|
334
|
+
residentRows: rows,
|
|
320
335
|
sourceLabel: dataset.sourceLabel ?? null,
|
|
321
336
|
tableNamespace: dataset.tableNamespace ?? null,
|
|
322
337
|
workProgress: dataset._metadata?.workProgress,
|
|
@@ -355,6 +370,7 @@ export function deserializeLegacyPlayDataset<T>(
|
|
|
355
370
|
count: dataset.count,
|
|
356
371
|
backing: dataset.backing,
|
|
357
372
|
previewRows: rows,
|
|
373
|
+
residentRows: complete ? rows : null,
|
|
358
374
|
sourceLabel: dataset.sourceLabel ?? null,
|
|
359
375
|
tableNamespace: dataset.tableNamespace ?? null,
|
|
360
376
|
workProgress: dataset._metadata?.workProgress,
|
|
@@ -404,14 +420,18 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
404
420
|
private readonly previewColumns?: string[];
|
|
405
421
|
private readonly workProgress?: PlayDatasetWorkProgressSummary;
|
|
406
422
|
private cachedCount: number;
|
|
423
|
+
private knownCount: number | null;
|
|
424
|
+
private readonly residentRows: readonly T[] | null;
|
|
407
425
|
private readonly resolvers: PlayDatasetResolvers<T>;
|
|
408
426
|
|
|
409
427
|
constructor(input: {
|
|
410
428
|
datasetKind: PlayDatasetKind;
|
|
411
429
|
datasetId: string;
|
|
412
430
|
count: number;
|
|
431
|
+
knownCount?: number | null;
|
|
413
432
|
backing?: PlayDatasetBacking;
|
|
414
433
|
previewRows: readonly T[];
|
|
434
|
+
residentRows?: readonly T[] | null;
|
|
415
435
|
sourceLabel?: string | null;
|
|
416
436
|
tableNamespace?: string | null;
|
|
417
437
|
workProgress?: PlayDatasetWorkProgressSummary;
|
|
@@ -420,8 +440,11 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
420
440
|
this.datasetKind = input.datasetKind;
|
|
421
441
|
this.datasetId = input.datasetId;
|
|
422
442
|
this.cachedCount = input.count;
|
|
443
|
+
this.knownCount =
|
|
444
|
+
input.knownCount === undefined ? input.count : input.knownCount;
|
|
423
445
|
this.backing = input.backing;
|
|
424
446
|
this.previewRows = input.previewRows;
|
|
447
|
+
this.residentRows = input.residentRows ?? null;
|
|
425
448
|
this.previewColumns = inferPreviewColumns(this.previewRows);
|
|
426
449
|
this.sourceLabel = input.sourceLabel ?? null;
|
|
427
450
|
this.tableNamespace = input.tableNamespace ?? null;
|
|
@@ -431,6 +454,7 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
431
454
|
|
|
432
455
|
async count(): Promise<number> {
|
|
433
456
|
this.cachedCount = await this.resolvers.count();
|
|
457
|
+
this.knownCount = this.cachedCount;
|
|
434
458
|
return this.cachedCount;
|
|
435
459
|
}
|
|
436
460
|
|
|
@@ -441,6 +465,32 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
441
465
|
return await this.resolvers.peek(limit);
|
|
442
466
|
}
|
|
443
467
|
|
|
468
|
+
async first(): Promise<T | undefined> {
|
|
469
|
+
return await this.at(0);
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
async at(index: number): Promise<T | undefined> {
|
|
473
|
+
if (!Number.isFinite(index)) return undefined;
|
|
474
|
+
const integer = Math.trunc(index);
|
|
475
|
+
const count = integer < 0 ? await this.count() : this.knownCount;
|
|
476
|
+
const normalized =
|
|
477
|
+
integer < 0 && count !== null ? count + integer : integer;
|
|
478
|
+
if (normalized < 0 || (count !== null && normalized >= count)) {
|
|
479
|
+
return undefined;
|
|
480
|
+
}
|
|
481
|
+
if (this.residentRows) return this.residentRows[normalized];
|
|
482
|
+
if (normalized < this.previewRows.length) {
|
|
483
|
+
return this.previewRows[normalized];
|
|
484
|
+
}
|
|
485
|
+
if (this.resolvers.at) return await this.resolvers.at(normalized);
|
|
486
|
+
let current = 0;
|
|
487
|
+
for await (const row of this.resolvers.iterate()) {
|
|
488
|
+
if (current === normalized) return row;
|
|
489
|
+
current += 1;
|
|
490
|
+
}
|
|
491
|
+
return undefined;
|
|
492
|
+
}
|
|
493
|
+
|
|
444
494
|
map<U>(
|
|
445
495
|
mapper: (row: T, index: number) => U | Promise<U>,
|
|
446
496
|
options?: PlayDatasetTransformOptions,
|
|
@@ -666,6 +716,7 @@ function createTransformedPlayDataset<T, U>(
|
|
|
666
716
|
key: options?.key,
|
|
667
717
|
}),
|
|
668
718
|
count: 0,
|
|
719
|
+
knownCount: null,
|
|
669
720
|
backing: source.backing,
|
|
670
721
|
sourceLabel,
|
|
671
722
|
tableNamespace: options?.key ?? null,
|
|
@@ -687,17 +738,93 @@ export function createDeferredPlayDataset<T>(input: {
|
|
|
687
738
|
datasetKind: PlayDatasetKind;
|
|
688
739
|
datasetId: string;
|
|
689
740
|
count: number;
|
|
741
|
+
/** Authoritative synchronous count, or null when execution must determine it. */
|
|
742
|
+
knownCount?: number | null;
|
|
690
743
|
backing?: PlayDatasetBacking;
|
|
691
744
|
previewRows?: readonly T[];
|
|
745
|
+
/** Complete in-memory row set. Never pass a bounded preview here. */
|
|
746
|
+
residentRows?: readonly T[] | null;
|
|
692
747
|
sourceLabel?: string | null;
|
|
693
748
|
tableNamespace?: string | null;
|
|
694
749
|
workProgress?: PlayDatasetWorkProgressSummary;
|
|
695
750
|
resolvers: PlayDatasetResolvers<T>;
|
|
696
751
|
}): PlayDataset<T> {
|
|
697
|
-
|
|
752
|
+
if (
|
|
753
|
+
input.residentRows &&
|
|
754
|
+
input.knownCount !== null &&
|
|
755
|
+
input.residentRows.length !== (input.knownCount ?? input.count)
|
|
756
|
+
) {
|
|
757
|
+
throw new Error(
|
|
758
|
+
`Resident Dataset Handle ${input.datasetId} has ${input.residentRows.length} rows but declares ${input.knownCount ?? input.count}.`,
|
|
759
|
+
);
|
|
760
|
+
}
|
|
761
|
+
const target = new DeferredPlayDataset({
|
|
698
762
|
...input,
|
|
699
763
|
previewRows: input.previewRows ?? [],
|
|
700
764
|
});
|
|
765
|
+
const boundMethods = new Map<PropertyKey, unknown>();
|
|
766
|
+
const dataset = new Proxy(target, {
|
|
767
|
+
get(dataset, property) {
|
|
768
|
+
if (property === 'length') {
|
|
769
|
+
throw datasetAsyncOnlyError(
|
|
770
|
+
dataset,
|
|
771
|
+
'Dataset Handles do not expose synchronous .length.',
|
|
772
|
+
'Use await dataset.count().',
|
|
773
|
+
);
|
|
774
|
+
}
|
|
775
|
+
if (typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property)) {
|
|
776
|
+
throw datasetAsyncOnlyError(
|
|
777
|
+
dataset,
|
|
778
|
+
`Dataset Handles do not expose synchronous row indexing (${property}).`,
|
|
779
|
+
`Use await dataset.at(${property}) or await dataset.first().`,
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
if (property === Symbol.iterator) {
|
|
783
|
+
return () => {
|
|
784
|
+
throw datasetAsyncOnlyError(
|
|
785
|
+
dataset,
|
|
786
|
+
'Dataset Handles do not support synchronous iteration.',
|
|
787
|
+
'Use for await...of or await dataset.materialize(limit).',
|
|
788
|
+
);
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
const value = Reflect.get(dataset, property, dataset);
|
|
792
|
+
if (typeof value !== 'function') return value;
|
|
793
|
+
if (!boundMethods.has(property)) {
|
|
794
|
+
boundMethods.set(property, value.bind(dataset));
|
|
795
|
+
}
|
|
796
|
+
return boundMethods.get(property);
|
|
797
|
+
},
|
|
798
|
+
set(_dataset, property) {
|
|
799
|
+
if (
|
|
800
|
+
property === 'length' ||
|
|
801
|
+
(typeof property === 'string' && /^(0|[1-9]\d*)$/.test(property))
|
|
802
|
+
) {
|
|
803
|
+
throw datasetAsyncOnlyError(
|
|
804
|
+
target,
|
|
805
|
+
'Dataset Handles do not support synchronous array assignment.',
|
|
806
|
+
'Transform rows with dataset.map(...) or explicitly materialize a bounded array.',
|
|
807
|
+
);
|
|
808
|
+
}
|
|
809
|
+
return false;
|
|
810
|
+
},
|
|
811
|
+
});
|
|
812
|
+
if (input.residentRows) {
|
|
813
|
+
residentRowsByDataset.set(target, input.residentRows);
|
|
814
|
+
residentRowsByDataset.set(dataset, input.residentRows);
|
|
815
|
+
}
|
|
816
|
+
return dataset;
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
function datasetAsyncOnlyError(
|
|
820
|
+
dataset: Pick<PlayDataset<unknown>, 'datasetId' | 'sourceLabel'>,
|
|
821
|
+
detail: string,
|
|
822
|
+
guidance: string,
|
|
823
|
+
): Error {
|
|
824
|
+
return new Error(
|
|
825
|
+
`PLAY_DATASET_ASYNC_ONLY: ${detail} ` +
|
|
826
|
+
`Dataset Handle ${dataset.sourceLabel ?? dataset.datasetId}. ${guidance}`,
|
|
827
|
+
);
|
|
701
828
|
}
|
|
702
829
|
|
|
703
830
|
export function createPlayDataset<T>(
|
|
@@ -719,6 +846,7 @@ export function createPlayDataset<T>(
|
|
|
719
846
|
`${metadata?.kind ?? 'map'}:${metadata?.tableNamespace ?? metadata?.sourceLabel ?? 'inline'}`,
|
|
720
847
|
count: materializedRows.length,
|
|
721
848
|
previewRows: materializedRows.slice(0, 5),
|
|
849
|
+
residentRows: materializedRows,
|
|
722
850
|
sourceLabel: metadata?.sourceLabel ?? null,
|
|
723
851
|
tableNamespace: metadata?.tableNamespace ?? null,
|
|
724
852
|
resolvers: {
|
package/dist/cli/index.js
CHANGED
|
@@ -1037,7 +1037,10 @@ var SDK_RELEASE = {
|
|
|
1037
1037
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
1038
1038
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
1039
1039
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
1040
|
-
|
|
1040
|
+
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
1041
|
+
// exposed storage-dependent synchronous access. This deliberate minor
|
|
1042
|
+
// release keeps lazy paging semantics independent of row residency.
|
|
1043
|
+
version: "0.2.0",
|
|
1041
1044
|
contracts: {
|
|
1042
1045
|
api: {
|
|
1043
1046
|
name: "sdk-http-api",
|
|
@@ -11991,7 +11994,7 @@ async function traceCliSpan(phase, fields, run) {
|
|
|
11991
11994
|
|
|
11992
11995
|
// src/cli/play-check-hints.ts
|
|
11993
11996
|
var EXTRACTED_GETTER_ERROR_HINT = "Deepline hint: extractedValues/extractedLists .get() only works for declared Deepline getters listed by `deepline tools describe <tool> --json`. Use `toolExecutionResult.toolResponse.raw` for provider/tool-specific scalar fields, and `toolExecutionResult.extractedLists.<name>.get()` for declared row/list outputs.";
|
|
11994
|
-
var DATASET_API_HINT = "Deepline hint: PlayDataset is lazy and
|
|
11997
|
+
var DATASET_API_HINT = "Deepline hint: PlayDataset is lazy, durable, and async-only. Use `.count()`, `.first()`, `.at(index)`, `.peek(n)`, `.materialize(limit)`, or async iteration; do not use `.length`, numeric indexing, spread, synchronous iteration, `.rows`, or `.toArray()`.";
|
|
11995
11998
|
var ROW_PROPERTY_HINT = "Deepline hint: this row type only contains fields produced by the CSV/schema and previous map steps. Check source column casing and the exact output field names from earlier steps before scaling.";
|
|
11996
11999
|
var TOOLS_EXECUTE_SIGNATURE_HINT = "Deepline hint: ctx.tools.execute requires a request object: `ctx.tools.execute({ id, tool, input, description })`. The stable `id` is required for logs, metadata, and receipt attachment; provider-call reuse is based on play, tool, semantic input, auth scope, provider action version, and cache policy.";
|
|
11997
12000
|
var RUN_PLAY_SIGNATURE_HINT = "Deepline hint: ctx.runPlay uses a stable key plus a composable child play reference. Direct-run-only or dataset-backed batch plays must be run directly, exported, then consumed by a separate play.";
|
|
@@ -12009,7 +12012,7 @@ function looksLikeInvalidExtractedGetter(error, sourceLine) {
|
|
|
12009
12012
|
function looksLikeDatasetApiMisuse(error, sourceLine) {
|
|
12010
12013
|
return /Property '(?:rows|toArray|forEach|map|filter|reduce)' does not exist on type '[^']*PlayDataset/.test(
|
|
12011
12014
|
error
|
|
12012
|
-
) || /\b(?:\.rows|\.toArray\(\)|\.forEach\(|\.map\(|\.filter\(|\.reduce\()/.test(
|
|
12015
|
+
) || /PlayDataset/.test(error) && (/Property 'length' does not exist/.test(error) || /can't be used to index type/.test(error) || /\[Symbol\.iterator\]/.test(error) || /not iterable/.test(error)) || /\b(?:\.rows|\.toArray\(\)|\.forEach\(|\.map\(|\.filter\(|\.reduce\()/.test(
|
|
12013
12016
|
sourceLine
|
|
12014
12017
|
);
|
|
12015
12018
|
}
|
package/dist/cli/index.mjs
CHANGED
|
@@ -1022,7 +1022,10 @@ var SDK_RELEASE = {
|
|
|
1022
1022
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
1023
1023
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
1024
1024
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
1025
|
-
|
|
1025
|
+
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
1026
|
+
// exposed storage-dependent synchronous access. This deliberate minor
|
|
1027
|
+
// release keeps lazy paging semantics independent of row residency.
|
|
1028
|
+
version: "0.2.0",
|
|
1026
1029
|
contracts: {
|
|
1027
1030
|
api: {
|
|
1028
1031
|
name: "sdk-http-api",
|
|
@@ -12020,7 +12023,7 @@ async function traceCliSpan(phase, fields, run) {
|
|
|
12020
12023
|
|
|
12021
12024
|
// src/cli/play-check-hints.ts
|
|
12022
12025
|
var EXTRACTED_GETTER_ERROR_HINT = "Deepline hint: extractedValues/extractedLists .get() only works for declared Deepline getters listed by `deepline tools describe <tool> --json`. Use `toolExecutionResult.toolResponse.raw` for provider/tool-specific scalar fields, and `toolExecutionResult.extractedLists.<name>.get()` for declared row/list outputs.";
|
|
12023
|
-
var DATASET_API_HINT = "Deepline hint: PlayDataset is lazy and
|
|
12026
|
+
var DATASET_API_HINT = "Deepline hint: PlayDataset is lazy, durable, and async-only. Use `.count()`, `.first()`, `.at(index)`, `.peek(n)`, `.materialize(limit)`, or async iteration; do not use `.length`, numeric indexing, spread, synchronous iteration, `.rows`, or `.toArray()`.";
|
|
12024
12027
|
var ROW_PROPERTY_HINT = "Deepline hint: this row type only contains fields produced by the CSV/schema and previous map steps. Check source column casing and the exact output field names from earlier steps before scaling.";
|
|
12025
12028
|
var TOOLS_EXECUTE_SIGNATURE_HINT = "Deepline hint: ctx.tools.execute requires a request object: `ctx.tools.execute({ id, tool, input, description })`. The stable `id` is required for logs, metadata, and receipt attachment; provider-call reuse is based on play, tool, semantic input, auth scope, provider action version, and cache policy.";
|
|
12026
12029
|
var RUN_PLAY_SIGNATURE_HINT = "Deepline hint: ctx.runPlay uses a stable key plus a composable child play reference. Direct-run-only or dataset-backed batch plays must be run directly, exported, then consumed by a separate play.";
|
|
@@ -12038,7 +12041,7 @@ function looksLikeInvalidExtractedGetter(error, sourceLine) {
|
|
|
12038
12041
|
function looksLikeDatasetApiMisuse(error, sourceLine) {
|
|
12039
12042
|
return /Property '(?:rows|toArray|forEach|map|filter|reduce)' does not exist on type '[^']*PlayDataset/.test(
|
|
12040
12043
|
error
|
|
12041
|
-
) || /\b(?:\.rows|\.toArray\(\)|\.forEach\(|\.map\(|\.filter\(|\.reduce\()/.test(
|
|
12044
|
+
) || /PlayDataset/.test(error) && (/Property 'length' does not exist/.test(error) || /can't be used to index type/.test(error) || /\[Symbol\.iterator\]/.test(error) || /not iterable/.test(error)) || /\b(?:\.rows|\.toArray\(\)|\.forEach\(|\.map\(|\.filter\(|\.reduce\()/.test(
|
|
12042
12045
|
sourceLine
|
|
12043
12046
|
);
|
|
12044
12047
|
}
|
package/dist/index.d.mts
CHANGED
|
@@ -3605,8 +3605,9 @@ type PlayDatasetTransformOptions = {
|
|
|
3605
3605
|
* runtime control. Use `count()` and `peek()` for bounded inspection. Use
|
|
3606
3606
|
* `materialize(limit)` or async iteration only when the dataset is intentionally
|
|
3607
3607
|
* small and bounded. `PlayDataset` intentionally does not expose `.rows`,
|
|
3608
|
-
* `.toArray()`,
|
|
3609
|
-
* persisted rows into memory
|
|
3608
|
+
* `.toArray()`, `.length`, numeric indexing, spread, or synchronous iteration;
|
|
3609
|
+
* those hide the runtime cost of loading persisted rows into memory or make
|
|
3610
|
+
* behavior depend on whether rows happen to be resident.
|
|
3610
3611
|
*
|
|
3611
3612
|
* @sdkReference runtime 190
|
|
3612
3613
|
*/
|
|
@@ -3626,6 +3627,10 @@ interface PlayDataset<T> extends AsyncIterable<T> {
|
|
|
3626
3627
|
count(): Promise<number>;
|
|
3627
3628
|
/** Preview rows. */
|
|
3628
3629
|
peek(limit?: number): Promise<T[]>;
|
|
3630
|
+
/** First row, loading it asynchronously when necessary. */
|
|
3631
|
+
first(): Promise<T | undefined>;
|
|
3632
|
+
/** Row at an array-style index, loading it asynchronously when necessary. */
|
|
3633
|
+
at(index: number): Promise<T | undefined>;
|
|
3629
3634
|
map<U>(mapper: (row: T, index: number) => U | Promise<U>, options?: PlayDatasetTransformOptions): PlayDataset<U>;
|
|
3630
3635
|
filter(predicate: (row: T, index: number) => boolean | Promise<boolean>, options?: PlayDatasetTransformOptions): PlayDataset<T>;
|
|
3631
3636
|
slice(start?: number, end?: number, options?: PlayDatasetTransformOptions): PlayDataset<T>;
|
package/dist/index.d.ts
CHANGED
|
@@ -3605,8 +3605,9 @@ type PlayDatasetTransformOptions = {
|
|
|
3605
3605
|
* runtime control. Use `count()` and `peek()` for bounded inspection. Use
|
|
3606
3606
|
* `materialize(limit)` or async iteration only when the dataset is intentionally
|
|
3607
3607
|
* small and bounded. `PlayDataset` intentionally does not expose `.rows`,
|
|
3608
|
-
* `.toArray()`,
|
|
3609
|
-
* persisted rows into memory
|
|
3608
|
+
* `.toArray()`, `.length`, numeric indexing, spread, or synchronous iteration;
|
|
3609
|
+
* those hide the runtime cost of loading persisted rows into memory or make
|
|
3610
|
+
* behavior depend on whether rows happen to be resident.
|
|
3610
3611
|
*
|
|
3611
3612
|
* @sdkReference runtime 190
|
|
3612
3613
|
*/
|
|
@@ -3626,6 +3627,10 @@ interface PlayDataset<T> extends AsyncIterable<T> {
|
|
|
3626
3627
|
count(): Promise<number>;
|
|
3627
3628
|
/** Preview rows. */
|
|
3628
3629
|
peek(limit?: number): Promise<T[]>;
|
|
3630
|
+
/** First row, loading it asynchronously when necessary. */
|
|
3631
|
+
first(): Promise<T | undefined>;
|
|
3632
|
+
/** Row at an array-style index, loading it asynchronously when necessary. */
|
|
3633
|
+
at(index: number): Promise<T | undefined>;
|
|
3629
3634
|
map<U>(mapper: (row: T, index: number) => U | Promise<U>, options?: PlayDatasetTransformOptions): PlayDataset<U>;
|
|
3630
3635
|
filter(predicate: (row: T, index: number) => boolean | Promise<boolean>, options?: PlayDatasetTransformOptions): PlayDataset<T>;
|
|
3631
3636
|
slice(start?: number, end?: number, options?: PlayDatasetTransformOptions): PlayDataset<T>;
|
package/dist/index.js
CHANGED
|
@@ -760,7 +760,10 @@ var SDK_RELEASE = {
|
|
|
760
760
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
761
761
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
762
762
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
763
|
-
|
|
763
|
+
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
764
|
+
// exposed storage-dependent synchronous access. This deliberate minor
|
|
765
|
+
// release keeps lazy paging semantics independent of row residency.
|
|
766
|
+
version: "0.2.0",
|
|
764
767
|
contracts: {
|
|
765
768
|
api: {
|
|
766
769
|
name: "sdk-http-api",
|
|
@@ -5790,6 +5793,7 @@ function isDeeplineExtractorTarget(value) {
|
|
|
5790
5793
|
// ../shared_libs/plays/dataset.ts
|
|
5791
5794
|
var PLAY_DATASET_BRAND = /* @__PURE__ */ Symbol.for("deepline.play.dataset");
|
|
5792
5795
|
var NODE_INSPECT_CUSTOM = /* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom");
|
|
5796
|
+
var residentRowsByDataset = /* @__PURE__ */ new WeakMap();
|
|
5793
5797
|
var DEFAULT_MATERIALIZE_LIMIT = 1e4;
|
|
5794
5798
|
var PLAY_DATASET_EXECUTION_PAGE_BYTES = 64 * 1024 * 1024;
|
|
5795
5799
|
var PLAY_DATASET_CELL_MAX_BYTES = 5 * 1024 * 1024;
|
|
@@ -5824,13 +5828,17 @@ var DeferredPlayDataset = class {
|
|
|
5824
5828
|
previewColumns;
|
|
5825
5829
|
workProgress;
|
|
5826
5830
|
cachedCount;
|
|
5831
|
+
knownCount;
|
|
5832
|
+
residentRows;
|
|
5827
5833
|
resolvers;
|
|
5828
5834
|
constructor(input) {
|
|
5829
5835
|
this.datasetKind = input.datasetKind;
|
|
5830
5836
|
this.datasetId = input.datasetId;
|
|
5831
5837
|
this.cachedCount = input.count;
|
|
5838
|
+
this.knownCount = input.knownCount === void 0 ? input.count : input.knownCount;
|
|
5832
5839
|
this.backing = input.backing;
|
|
5833
5840
|
this.previewRows = input.previewRows;
|
|
5841
|
+
this.residentRows = input.residentRows ?? null;
|
|
5834
5842
|
this.previewColumns = inferPreviewColumns(this.previewRows);
|
|
5835
5843
|
this.sourceLabel = input.sourceLabel ?? null;
|
|
5836
5844
|
this.tableNamespace = input.tableNamespace ?? null;
|
|
@@ -5839,6 +5847,7 @@ var DeferredPlayDataset = class {
|
|
|
5839
5847
|
}
|
|
5840
5848
|
async count() {
|
|
5841
5849
|
this.cachedCount = await this.resolvers.count();
|
|
5850
|
+
this.knownCount = this.cachedCount;
|
|
5842
5851
|
return this.cachedCount;
|
|
5843
5852
|
}
|
|
5844
5853
|
async peek(limit = 10) {
|
|
@@ -5847,6 +5856,29 @@ var DeferredPlayDataset = class {
|
|
|
5847
5856
|
}
|
|
5848
5857
|
return await this.resolvers.peek(limit);
|
|
5849
5858
|
}
|
|
5859
|
+
async first() {
|
|
5860
|
+
return await this.at(0);
|
|
5861
|
+
}
|
|
5862
|
+
async at(index) {
|
|
5863
|
+
if (!Number.isFinite(index)) return void 0;
|
|
5864
|
+
const integer = Math.trunc(index);
|
|
5865
|
+
const count = integer < 0 ? await this.count() : this.knownCount;
|
|
5866
|
+
const normalized = integer < 0 && count !== null ? count + integer : integer;
|
|
5867
|
+
if (normalized < 0 || count !== null && normalized >= count) {
|
|
5868
|
+
return void 0;
|
|
5869
|
+
}
|
|
5870
|
+
if (this.residentRows) return this.residentRows[normalized];
|
|
5871
|
+
if (normalized < this.previewRows.length) {
|
|
5872
|
+
return this.previewRows[normalized];
|
|
5873
|
+
}
|
|
5874
|
+
if (this.resolvers.at) return await this.resolvers.at(normalized);
|
|
5875
|
+
let current = 0;
|
|
5876
|
+
for await (const row of this.resolvers.iterate()) {
|
|
5877
|
+
if (current === normalized) return row;
|
|
5878
|
+
current += 1;
|
|
5879
|
+
}
|
|
5880
|
+
return void 0;
|
|
5881
|
+
}
|
|
5850
5882
|
map(mapper, options) {
|
|
5851
5883
|
return createTransformedPlayDataset(this, { kind: "map", mapper }, options);
|
|
5852
5884
|
}
|
|
@@ -6009,6 +6041,7 @@ function createTransformedPlayDataset(source, transform, options) {
|
|
|
6009
6041
|
key: options?.key
|
|
6010
6042
|
}),
|
|
6011
6043
|
count: 0,
|
|
6044
|
+
knownCount: null,
|
|
6012
6045
|
backing: source.backing,
|
|
6013
6046
|
sourceLabel,
|
|
6014
6047
|
tableNamespace: options?.key ?? null,
|
|
@@ -6025,10 +6058,69 @@ function createTransformedPlayDataset(source, transform, options) {
|
|
|
6025
6058
|
});
|
|
6026
6059
|
}
|
|
6027
6060
|
function createDeferredPlayDataset(input) {
|
|
6028
|
-
|
|
6061
|
+
if (input.residentRows && input.knownCount !== null && input.residentRows.length !== (input.knownCount ?? input.count)) {
|
|
6062
|
+
throw new Error(
|
|
6063
|
+
`Resident Dataset Handle ${input.datasetId} has ${input.residentRows.length} rows but declares ${input.knownCount ?? input.count}.`
|
|
6064
|
+
);
|
|
6065
|
+
}
|
|
6066
|
+
const target = new DeferredPlayDataset({
|
|
6029
6067
|
...input,
|
|
6030
6068
|
previewRows: input.previewRows ?? []
|
|
6031
6069
|
});
|
|
6070
|
+
const boundMethods = /* @__PURE__ */ new Map();
|
|
6071
|
+
const dataset = new Proxy(target, {
|
|
6072
|
+
get(dataset2, property) {
|
|
6073
|
+
if (property === "length") {
|
|
6074
|
+
throw datasetAsyncOnlyError(
|
|
6075
|
+
dataset2,
|
|
6076
|
+
"Dataset Handles do not expose synchronous .length.",
|
|
6077
|
+
"Use await dataset.count()."
|
|
6078
|
+
);
|
|
6079
|
+
}
|
|
6080
|
+
if (typeof property === "string" && /^(0|[1-9]\d*)$/.test(property)) {
|
|
6081
|
+
throw datasetAsyncOnlyError(
|
|
6082
|
+
dataset2,
|
|
6083
|
+
`Dataset Handles do not expose synchronous row indexing (${property}).`,
|
|
6084
|
+
`Use await dataset.at(${property}) or await dataset.first().`
|
|
6085
|
+
);
|
|
6086
|
+
}
|
|
6087
|
+
if (property === Symbol.iterator) {
|
|
6088
|
+
return () => {
|
|
6089
|
+
throw datasetAsyncOnlyError(
|
|
6090
|
+
dataset2,
|
|
6091
|
+
"Dataset Handles do not support synchronous iteration.",
|
|
6092
|
+
"Use for await...of or await dataset.materialize(limit)."
|
|
6093
|
+
);
|
|
6094
|
+
};
|
|
6095
|
+
}
|
|
6096
|
+
const value = Reflect.get(dataset2, property, dataset2);
|
|
6097
|
+
if (typeof value !== "function") return value;
|
|
6098
|
+
if (!boundMethods.has(property)) {
|
|
6099
|
+
boundMethods.set(property, value.bind(dataset2));
|
|
6100
|
+
}
|
|
6101
|
+
return boundMethods.get(property);
|
|
6102
|
+
},
|
|
6103
|
+
set(_dataset, property) {
|
|
6104
|
+
if (property === "length" || typeof property === "string" && /^(0|[1-9]\d*)$/.test(property)) {
|
|
6105
|
+
throw datasetAsyncOnlyError(
|
|
6106
|
+
target,
|
|
6107
|
+
"Dataset Handles do not support synchronous array assignment.",
|
|
6108
|
+
"Transform rows with dataset.map(...) or explicitly materialize a bounded array."
|
|
6109
|
+
);
|
|
6110
|
+
}
|
|
6111
|
+
return false;
|
|
6112
|
+
}
|
|
6113
|
+
});
|
|
6114
|
+
if (input.residentRows) {
|
|
6115
|
+
residentRowsByDataset.set(target, input.residentRows);
|
|
6116
|
+
residentRowsByDataset.set(dataset, input.residentRows);
|
|
6117
|
+
}
|
|
6118
|
+
return dataset;
|
|
6119
|
+
}
|
|
6120
|
+
function datasetAsyncOnlyError(dataset, detail, guidance) {
|
|
6121
|
+
return new Error(
|
|
6122
|
+
`PLAY_DATASET_ASYNC_ONLY: ${detail} Dataset Handle ${dataset.sourceLabel ?? dataset.datasetId}. ${guidance}`
|
|
6123
|
+
);
|
|
6032
6124
|
}
|
|
6033
6125
|
function createPlayDataset(rows, metadata) {
|
|
6034
6126
|
const materializedRows = [...rows];
|
|
@@ -6037,6 +6129,7 @@ function createPlayDataset(rows, metadata) {
|
|
|
6037
6129
|
datasetId: metadata?.datasetId ?? `${metadata?.kind ?? "map"}:${metadata?.tableNamespace ?? metadata?.sourceLabel ?? "inline"}`,
|
|
6038
6130
|
count: materializedRows.length,
|
|
6039
6131
|
previewRows: materializedRows.slice(0, 5),
|
|
6132
|
+
residentRows: materializedRows,
|
|
6040
6133
|
sourceLabel: metadata?.sourceLabel ?? null,
|
|
6041
6134
|
tableNamespace: metadata?.tableNamespace ?? null,
|
|
6042
6135
|
resolvers: {
|
|
@@ -7346,7 +7439,9 @@ function attachSdkQueryResultDatasetResult(toolId, result, options) {
|
|
|
7346
7439
|
return result;
|
|
7347
7440
|
}
|
|
7348
7441
|
const datasetLimit = finitePositiveInteger(dataset.returned_limit) ?? totalRows;
|
|
7349
|
-
const
|
|
7442
|
+
const effectiveCount = Math.min(totalRows, datasetLimit);
|
|
7443
|
+
const rawRows = rowsFromUnknown(raw?.rows);
|
|
7444
|
+
const previewRows = rawRows.slice(0, 25);
|
|
7350
7445
|
const fetchPage = async (offset, limit) => {
|
|
7351
7446
|
if (limit <= 0 || offset >= totalRows) return [];
|
|
7352
7447
|
const response = await options.client.executeTool(toolId, options.input, {
|
|
@@ -7393,10 +7488,12 @@ function attachSdkQueryResultDatasetResult(toolId, result, options) {
|
|
|
7393
7488
|
datasetId: `sdk-tool-list:${toolId}:${stableHash(datasetScope)}:${datasetLimit}:${safeNonce}`,
|
|
7394
7489
|
count: Math.min(totalRows, datasetLimit),
|
|
7395
7490
|
previewRows,
|
|
7491
|
+
residentRows: rawRows.length >= effectiveCount ? rawRows.slice(0, effectiveCount) : null,
|
|
7396
7492
|
sourceLabel: "query result rows",
|
|
7397
7493
|
tableNamespace: null,
|
|
7398
7494
|
resolvers: {
|
|
7399
7495
|
count: async () => Math.min(totalRows, datasetLimit),
|
|
7496
|
+
at: async (index) => (await fetchPage(index, 1))[0],
|
|
7400
7497
|
peek: async (limit) => collectRows(limit),
|
|
7401
7498
|
materialize: async (limit) => collectRows(limit),
|
|
7402
7499
|
iterate: () => ({
|
package/dist/index.mjs
CHANGED
|
@@ -686,7 +686,10 @@ var SDK_RELEASE = {
|
|
|
686
686
|
// 0.1.253 makes play-page browser opening opt-in and retires --no-open.
|
|
687
687
|
// 0.1.254 removes the internal operations tree from the published SDK CLI.
|
|
688
688
|
// Operators use the checkout-local deepline-admin binary instead.
|
|
689
|
-
|
|
689
|
+
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
690
|
+
// exposed storage-dependent synchronous access. This deliberate minor
|
|
691
|
+
// release keeps lazy paging semantics independent of row residency.
|
|
692
|
+
version: "0.2.0",
|
|
690
693
|
contracts: {
|
|
691
694
|
api: {
|
|
692
695
|
name: "sdk-http-api",
|
|
@@ -5716,6 +5719,7 @@ function isDeeplineExtractorTarget(value) {
|
|
|
5716
5719
|
// ../shared_libs/plays/dataset.ts
|
|
5717
5720
|
var PLAY_DATASET_BRAND = /* @__PURE__ */ Symbol.for("deepline.play.dataset");
|
|
5718
5721
|
var NODE_INSPECT_CUSTOM = /* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom");
|
|
5722
|
+
var residentRowsByDataset = /* @__PURE__ */ new WeakMap();
|
|
5719
5723
|
var DEFAULT_MATERIALIZE_LIMIT = 1e4;
|
|
5720
5724
|
var PLAY_DATASET_EXECUTION_PAGE_BYTES = 64 * 1024 * 1024;
|
|
5721
5725
|
var PLAY_DATASET_CELL_MAX_BYTES = 5 * 1024 * 1024;
|
|
@@ -5750,13 +5754,17 @@ var DeferredPlayDataset = class {
|
|
|
5750
5754
|
previewColumns;
|
|
5751
5755
|
workProgress;
|
|
5752
5756
|
cachedCount;
|
|
5757
|
+
knownCount;
|
|
5758
|
+
residentRows;
|
|
5753
5759
|
resolvers;
|
|
5754
5760
|
constructor(input) {
|
|
5755
5761
|
this.datasetKind = input.datasetKind;
|
|
5756
5762
|
this.datasetId = input.datasetId;
|
|
5757
5763
|
this.cachedCount = input.count;
|
|
5764
|
+
this.knownCount = input.knownCount === void 0 ? input.count : input.knownCount;
|
|
5758
5765
|
this.backing = input.backing;
|
|
5759
5766
|
this.previewRows = input.previewRows;
|
|
5767
|
+
this.residentRows = input.residentRows ?? null;
|
|
5760
5768
|
this.previewColumns = inferPreviewColumns(this.previewRows);
|
|
5761
5769
|
this.sourceLabel = input.sourceLabel ?? null;
|
|
5762
5770
|
this.tableNamespace = input.tableNamespace ?? null;
|
|
@@ -5765,6 +5773,7 @@ var DeferredPlayDataset = class {
|
|
|
5765
5773
|
}
|
|
5766
5774
|
async count() {
|
|
5767
5775
|
this.cachedCount = await this.resolvers.count();
|
|
5776
|
+
this.knownCount = this.cachedCount;
|
|
5768
5777
|
return this.cachedCount;
|
|
5769
5778
|
}
|
|
5770
5779
|
async peek(limit = 10) {
|
|
@@ -5773,6 +5782,29 @@ var DeferredPlayDataset = class {
|
|
|
5773
5782
|
}
|
|
5774
5783
|
return await this.resolvers.peek(limit);
|
|
5775
5784
|
}
|
|
5785
|
+
async first() {
|
|
5786
|
+
return await this.at(0);
|
|
5787
|
+
}
|
|
5788
|
+
async at(index) {
|
|
5789
|
+
if (!Number.isFinite(index)) return void 0;
|
|
5790
|
+
const integer = Math.trunc(index);
|
|
5791
|
+
const count = integer < 0 ? await this.count() : this.knownCount;
|
|
5792
|
+
const normalized = integer < 0 && count !== null ? count + integer : integer;
|
|
5793
|
+
if (normalized < 0 || count !== null && normalized >= count) {
|
|
5794
|
+
return void 0;
|
|
5795
|
+
}
|
|
5796
|
+
if (this.residentRows) return this.residentRows[normalized];
|
|
5797
|
+
if (normalized < this.previewRows.length) {
|
|
5798
|
+
return this.previewRows[normalized];
|
|
5799
|
+
}
|
|
5800
|
+
if (this.resolvers.at) return await this.resolvers.at(normalized);
|
|
5801
|
+
let current = 0;
|
|
5802
|
+
for await (const row of this.resolvers.iterate()) {
|
|
5803
|
+
if (current === normalized) return row;
|
|
5804
|
+
current += 1;
|
|
5805
|
+
}
|
|
5806
|
+
return void 0;
|
|
5807
|
+
}
|
|
5776
5808
|
map(mapper, options) {
|
|
5777
5809
|
return createTransformedPlayDataset(this, { kind: "map", mapper }, options);
|
|
5778
5810
|
}
|
|
@@ -5935,6 +5967,7 @@ function createTransformedPlayDataset(source, transform, options) {
|
|
|
5935
5967
|
key: options?.key
|
|
5936
5968
|
}),
|
|
5937
5969
|
count: 0,
|
|
5970
|
+
knownCount: null,
|
|
5938
5971
|
backing: source.backing,
|
|
5939
5972
|
sourceLabel,
|
|
5940
5973
|
tableNamespace: options?.key ?? null,
|
|
@@ -5951,10 +5984,69 @@ function createTransformedPlayDataset(source, transform, options) {
|
|
|
5951
5984
|
});
|
|
5952
5985
|
}
|
|
5953
5986
|
function createDeferredPlayDataset(input) {
|
|
5954
|
-
|
|
5987
|
+
if (input.residentRows && input.knownCount !== null && input.residentRows.length !== (input.knownCount ?? input.count)) {
|
|
5988
|
+
throw new Error(
|
|
5989
|
+
`Resident Dataset Handle ${input.datasetId} has ${input.residentRows.length} rows but declares ${input.knownCount ?? input.count}.`
|
|
5990
|
+
);
|
|
5991
|
+
}
|
|
5992
|
+
const target = new DeferredPlayDataset({
|
|
5955
5993
|
...input,
|
|
5956
5994
|
previewRows: input.previewRows ?? []
|
|
5957
5995
|
});
|
|
5996
|
+
const boundMethods = /* @__PURE__ */ new Map();
|
|
5997
|
+
const dataset = new Proxy(target, {
|
|
5998
|
+
get(dataset2, property) {
|
|
5999
|
+
if (property === "length") {
|
|
6000
|
+
throw datasetAsyncOnlyError(
|
|
6001
|
+
dataset2,
|
|
6002
|
+
"Dataset Handles do not expose synchronous .length.",
|
|
6003
|
+
"Use await dataset.count()."
|
|
6004
|
+
);
|
|
6005
|
+
}
|
|
6006
|
+
if (typeof property === "string" && /^(0|[1-9]\d*)$/.test(property)) {
|
|
6007
|
+
throw datasetAsyncOnlyError(
|
|
6008
|
+
dataset2,
|
|
6009
|
+
`Dataset Handles do not expose synchronous row indexing (${property}).`,
|
|
6010
|
+
`Use await dataset.at(${property}) or await dataset.first().`
|
|
6011
|
+
);
|
|
6012
|
+
}
|
|
6013
|
+
if (property === Symbol.iterator) {
|
|
6014
|
+
return () => {
|
|
6015
|
+
throw datasetAsyncOnlyError(
|
|
6016
|
+
dataset2,
|
|
6017
|
+
"Dataset Handles do not support synchronous iteration.",
|
|
6018
|
+
"Use for await...of or await dataset.materialize(limit)."
|
|
6019
|
+
);
|
|
6020
|
+
};
|
|
6021
|
+
}
|
|
6022
|
+
const value = Reflect.get(dataset2, property, dataset2);
|
|
6023
|
+
if (typeof value !== "function") return value;
|
|
6024
|
+
if (!boundMethods.has(property)) {
|
|
6025
|
+
boundMethods.set(property, value.bind(dataset2));
|
|
6026
|
+
}
|
|
6027
|
+
return boundMethods.get(property);
|
|
6028
|
+
},
|
|
6029
|
+
set(_dataset, property) {
|
|
6030
|
+
if (property === "length" || typeof property === "string" && /^(0|[1-9]\d*)$/.test(property)) {
|
|
6031
|
+
throw datasetAsyncOnlyError(
|
|
6032
|
+
target,
|
|
6033
|
+
"Dataset Handles do not support synchronous array assignment.",
|
|
6034
|
+
"Transform rows with dataset.map(...) or explicitly materialize a bounded array."
|
|
6035
|
+
);
|
|
6036
|
+
}
|
|
6037
|
+
return false;
|
|
6038
|
+
}
|
|
6039
|
+
});
|
|
6040
|
+
if (input.residentRows) {
|
|
6041
|
+
residentRowsByDataset.set(target, input.residentRows);
|
|
6042
|
+
residentRowsByDataset.set(dataset, input.residentRows);
|
|
6043
|
+
}
|
|
6044
|
+
return dataset;
|
|
6045
|
+
}
|
|
6046
|
+
function datasetAsyncOnlyError(dataset, detail, guidance) {
|
|
6047
|
+
return new Error(
|
|
6048
|
+
`PLAY_DATASET_ASYNC_ONLY: ${detail} Dataset Handle ${dataset.sourceLabel ?? dataset.datasetId}. ${guidance}`
|
|
6049
|
+
);
|
|
5958
6050
|
}
|
|
5959
6051
|
function createPlayDataset(rows, metadata) {
|
|
5960
6052
|
const materializedRows = [...rows];
|
|
@@ -5963,6 +6055,7 @@ function createPlayDataset(rows, metadata) {
|
|
|
5963
6055
|
datasetId: metadata?.datasetId ?? `${metadata?.kind ?? "map"}:${metadata?.tableNamespace ?? metadata?.sourceLabel ?? "inline"}`,
|
|
5964
6056
|
count: materializedRows.length,
|
|
5965
6057
|
previewRows: materializedRows.slice(0, 5),
|
|
6058
|
+
residentRows: materializedRows,
|
|
5966
6059
|
sourceLabel: metadata?.sourceLabel ?? null,
|
|
5967
6060
|
tableNamespace: metadata?.tableNamespace ?? null,
|
|
5968
6061
|
resolvers: {
|
|
@@ -7272,7 +7365,9 @@ function attachSdkQueryResultDatasetResult(toolId, result, options) {
|
|
|
7272
7365
|
return result;
|
|
7273
7366
|
}
|
|
7274
7367
|
const datasetLimit = finitePositiveInteger(dataset.returned_limit) ?? totalRows;
|
|
7275
|
-
const
|
|
7368
|
+
const effectiveCount = Math.min(totalRows, datasetLimit);
|
|
7369
|
+
const rawRows = rowsFromUnknown(raw?.rows);
|
|
7370
|
+
const previewRows = rawRows.slice(0, 25);
|
|
7276
7371
|
const fetchPage = async (offset, limit) => {
|
|
7277
7372
|
if (limit <= 0 || offset >= totalRows) return [];
|
|
7278
7373
|
const response = await options.client.executeTool(toolId, options.input, {
|
|
@@ -7319,10 +7414,12 @@ function attachSdkQueryResultDatasetResult(toolId, result, options) {
|
|
|
7319
7414
|
datasetId: `sdk-tool-list:${toolId}:${stableHash(datasetScope)}:${datasetLimit}:${safeNonce}`,
|
|
7320
7415
|
count: Math.min(totalRows, datasetLimit),
|
|
7321
7416
|
previewRows,
|
|
7417
|
+
residentRows: rawRows.length >= effectiveCount ? rawRows.slice(0, effectiveCount) : null,
|
|
7322
7418
|
sourceLabel: "query result rows",
|
|
7323
7419
|
tableNamespace: null,
|
|
7324
7420
|
resolvers: {
|
|
7325
7421
|
count: async () => Math.min(totalRows, datasetLimit),
|
|
7422
|
+
at: async (index) => (await fetchPage(index, 1))[0],
|
|
7326
7423
|
peek: async (limit) => collectRows(limit),
|
|
7327
7424
|
materialize: async (limit) => collectRows(limit),
|
|
7328
7425
|
iterate: () => ({
|