@gscdump/engine 0.35.5 → 0.35.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/_chunks/sink.d.mts
CHANGED
|
@@ -2,7 +2,7 @@ import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mj
|
|
|
2
2
|
import { EngineError } from "./errors.mjs";
|
|
3
3
|
import { icebergCreateTable } from "./libs/icebird.mjs";
|
|
4
4
|
import { Result } from "gscdump/result";
|
|
5
|
-
import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, IcebergCatalogConfig, IcebergColumn, IcebergColumn as IcebergColumn$1, IcebergColumnType, IcebergConnection, IcebergConnection as IcebergConnection$1, IcebergListedDataFile, IcebergListedDataFile as IcebergListedDataFile$1, IcebergPartitionField, IcebergPartitionField as IcebergPartitionField$1, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableSpec, IcebergTableSpec as IcebergTableSpec$1, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1, QueryProfiler, connectIcebergCatalog, ensureIcebergNamespace, listIcebergTables } from "@gscdump/lakehouse";
|
|
5
|
+
import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, IcebergCatalogConfig, IcebergColumn, IcebergColumn as IcebergColumn$1, IcebergColumnType, IcebergConnection, IcebergConnection as IcebergConnection$1, IcebergListedDataFile, IcebergListedDataFile as IcebergListedDataFile$1, IcebergPartitionField, IcebergPartitionField as IcebergPartitionField$1, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableSpec, IcebergTableSpec as IcebergTableSpec$1, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1, QueryProfiler, connectIcebergCatalog, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables } from "@gscdump/lakehouse";
|
|
6
6
|
import { icebergAppendRetrying, isCommitRateLimited } from "@gscdump/lakehouse/unsafe-raw";
|
|
7
7
|
import { TableName } from "@gscdump/contracts";
|
|
8
8
|
/** The 9 fact tables that exist as global Iceberg tables. */
|
|
@@ -248,4 +248,4 @@ interface IcebergAppendSinkOptions extends SinkOptions {
|
|
|
248
248
|
*/
|
|
249
249
|
encoding?: PartitionKeyEncoding$1;
|
|
250
250
|
}
|
|
251
|
-
export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn$1 as IcebergColumn, type IcebergColumnType, type IcebergConnection$1 as IcebergConnection, type IcebergListedDataFile$1 as IcebergListedDataFile, type IcebergPartitionField$1 as IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, type IcebergTableSpec$1 as IcebergTableSpec, ListIcebergDataFilesOptions, type PartitionKeyEncoding$1 as PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables$1 as dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
|
|
251
|
+
export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn$1 as IcebergColumn, type IcebergColumnType, type IcebergConnection$1 as IcebergConnection, type IcebergListedDataFile$1 as IcebergListedDataFile, type IcebergPartitionField$1 as IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, type IcebergTableSpec$1 as IcebergTableSpec, ListIcebergDataFilesOptions, type PartitionKeyEncoding$1 as PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables$1 as dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, invalidateSnapshotRef, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
|
package/dist/iceberg/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
|
|
1
|
+
import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, invalidateSnapshotRef, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
|
|
2
2
|
import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
|
|
3
3
|
type IcebergAppendSink = Sink;
|
|
4
4
|
/**
|
|
@@ -10,4 +10,4 @@ type IcebergAppendSink = Sink;
|
|
|
10
10
|
* with no rows never touches the network.
|
|
11
11
|
*/
|
|
12
12
|
declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
|
|
13
|
-
export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
|
|
13
|
+
export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, invalidateSnapshotRef, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
|
package/dist/iceberg/index.mjs
CHANGED
|
@@ -3,7 +3,7 @@ import { engineErrors } from "../errors.mjs";
|
|
|
3
3
|
import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
|
|
4
4
|
import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
|
|
5
5
|
import { err, ok } from "gscdump/result";
|
|
6
|
-
import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables as dropIcebergTables$1, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
|
|
6
|
+
import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables as dropIcebergTables$1, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
|
|
7
7
|
import { icebergAppendRetrying, isCommitRateLimited } from "@gscdump/lakehouse/unsafe-raw";
|
|
8
8
|
function icebergSchemaFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
|
|
9
9
|
return {
|
|
@@ -238,4 +238,4 @@ function createIcebergAppendSink(options) {
|
|
|
238
238
|
}
|
|
239
239
|
};
|
|
240
240
|
}
|
|
241
|
-
export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
|
|
241
|
+
export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, invalidateSnapshotRef, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.35.
|
|
4
|
+
"version": "0.35.6",
|
|
5
5
|
"description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -181,9 +181,9 @@
|
|
|
181
181
|
"dependencies": {
|
|
182
182
|
"drizzle-orm": "1.0.0-rc.3",
|
|
183
183
|
"proper-lockfile": "^4.1.2",
|
|
184
|
-
"@gscdump/
|
|
185
|
-
"@gscdump/
|
|
186
|
-
"gscdump": "0.35.
|
|
184
|
+
"@gscdump/contracts": "0.35.6",
|
|
185
|
+
"@gscdump/lakehouse": "0.35.6",
|
|
186
|
+
"gscdump": "0.35.6"
|
|
187
187
|
},
|
|
188
188
|
"devDependencies": {
|
|
189
189
|
"@duckdb/duckdb-wasm": "^1.32.0",
|