node-s3tables 0.0.9 → 0.0.11

package/README.md

@@ -18,6 +18,7 @@ import {
   addManifest,
   addDataFiles,
   setCurrentCommit,
+  removeSnapshots,
 } from 'node-s3tables';
 
 // Get table metadata
@@ -184,6 +185,7 @@ Adds data files to an S3 table by creating a new snapshot.
 - `params.namespace` (string) - The namespace name
 - `params.name` (string) - The table name
 - `params.lists` (AddFileList[]) - Array of file lists to add
+- `params.maxSnapshots` (number, optional) - Maximum number of snapshots to retain. When set, automatically removes the oldest snapshot if the count exceeds this limit
 - `params.credentials` (AwsCredentialIdentity, optional) - AWS credentials
 
 **Returns:** Promise<string>
@@ -208,6 +210,28 @@ await addDataFiles({
     },
   ],
 });
+
+// With automatic snapshot cleanup
+await addDataFiles({
+  tableBucketARN: 'arn:aws:s3tables:us-west-2:123456789012:bucket/my-bucket',
+  namespace: 'sales',
+  name: 'daily_sales',
+  maxSnapshots: 10, // Keep only the 10 most recent snapshots
+  lists: [
+    {
+      specId: 1,
+      schemaId: 2,
+      files: [
+        {
+          file: 's3://my-bucket/data/sales-2024-01-02.parquet',
+          partitions: { sale_date_day: '2024-01-02' },
+          recordCount: 1500n,
+          fileSize: 78643n,
+        },
+      ],
+    },
+  ],
+});
 ```
 
 ### setCurrentCommit(params)
@@ -233,6 +257,29 @@ await setCurrentCommit({
 });
 ```
 
+### removeSnapshots(params)
+
+Removes snapshots from an S3 table. Note: Due to Iceberg limitations, only one snapshot can be removed at a time.
+
+**Parameters:**
+
+- `params.tableBucketARN` (string) - The ARN of the table bucket
+- `params.namespace` (string) - The namespace name
+- `params.name` (string) - The table name
+- `params.snapshotIds` (bigint[]) - Array of snapshot IDs to remove (only provide one snapshot ID)
+- `params.credentials` (AwsCredentialIdentity, optional) - AWS credentials
+
+**Returns:** Promise<IcebergUpdateResponse>
+
+```javascript
+await removeSnapshots({
+  tableBucketARN: 'arn:aws:s3tables:us-west-2:123456789012:bucket/my-bucket',
+  namespace: 'sales',
+  name: 'daily_sales',
+  snapshotIds: [4183020680887155442n], // Only one snapshot ID
+});
+```
+
 ## Type Definitions
 
 ### IcebergUpdateResponse

package/dist/index.d.ts

@@ -20,9 +20,9 @@ interface ManifestListRecord {
     sequence_number: bigint;
     min_sequence_number: bigint;
     added_snapshot_id: bigint;
-    added_data_files_count: number;
-    existing_data_files_count: number;
-    deleted_data_files_count: number;
+    added_files_count: number;
+    existing_files_count: number;
+    deleted_files_count: number;
     added_rows_count: bigint;
     existing_rows_count: bigint;
     deleted_rows_count: bigint;
@@ -81,6 +81,7 @@ interface IcebergSnapshot {
 }
 interface IcebergMetadata {
     'last-column-id': number;
+    'last-sequence-number': bigint | number;
     'current-schema-id': number;
     schemas: IcebergSchema[];
     snapshots: IcebergSnapshot[];
@@ -153,6 +154,14 @@ interface AddPartitionSpecParams {
     fields: IcebergPartitionField[];
 }
 declare function addPartitionSpec(params: AddPartitionSpecParams): Promise<IcebergUpdateResponse>;
+interface RemoveSnapshotsParams {
+    credentials?: AwsCredentialIdentity;
+    tableBucketARN: string;
+    namespace: string;
+    name: string;
+    snapshotIds: bigint[];
+}
+declare function removeSnapshots(params: RemoveSnapshotsParams): Promise<IcebergUpdateResponse>;
 
 type JSONPrimitive = string | number | boolean | null | bigint | undefined;
 type JSONValue = JSONPrimitive | JSONObject | JSONArray;
@@ -174,6 +183,7 @@ interface AddDataFilesParams {
     snapshotId?: bigint;
     lists: AddFileList[];
     retryCount?: number;
+    maxSnapshots?: number;
 }
 interface AddDataFilesResult {
     result: JSONObject;
@@ -207,7 +217,8 @@ declare const _default: {
     addManifest: typeof addManifest;
     addDataFiles: typeof addDataFiles;
     setCurrentCommit: typeof setCurrentCommit;
+    removeSnapshots: typeof removeSnapshots;
 };
 
-export { IcebergHttpError, addDataFiles, addManifest, addPartitionSpec, addSchema, _default as default, getMetadata, setCurrentCommit };
-export type { AddDataFilesParams, AddDataFilesResult, AddFile, AddFileList, AddManifestParams, AddPartitionSpecParams, AddSchemaParams, GetMetadataParams, IcebergComplexType, IcebergMetadata, IcebergPartitionField, IcebergPartitionSpec, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSnapshot, IcebergSnapshotSummary, IcebergTransform, IcebergType, IcebergUpdateResponse, SetCurrentCommitParams, TableLocation };
+export { IcebergHttpError, addDataFiles, addManifest, addPartitionSpec, addSchema, _default as default, getMetadata, removeSnapshots, setCurrentCommit };
+export type { AddDataFilesParams, AddDataFilesResult, AddFile, AddFileList, AddManifestParams, AddPartitionSpecParams, AddSchemaParams, GetMetadataParams, IcebergComplexType, IcebergMetadata, IcebergPartitionField, IcebergPartitionSpec, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSnapshot, IcebergSnapshotSummary, IcebergTransform, IcebergType, IcebergUpdateResponse, RemoveSnapshotsParams, SetCurrentCommitParams, TableLocation };

package/dist/index.js

@@ -4,6 +4,7 @@ Object.defineProperty(exports, '__esModule', { value: true });
 
 var node_crypto = require('node:crypto');
 var avsc = require('avsc');
+var zlib = require('node:zlib');
 var clientS3 = require('@aws-sdk/client-s3');
 var clientS3tables = require('@aws-sdk/client-s3tables');
 var libStorage = require('@aws-sdk/lib-storage');
@@ -32,6 +33,7 @@ function _interopNamespaceDefault(e) {
 }
 
 var avsc__namespace = /*#__PURE__*/_interopNamespaceDefault(avsc);
+var zlib__namespace = /*#__PURE__*/_interopNamespaceDefault(zlib);
 var LosslessJson__namespace = /*#__PURE__*/_interopNamespaceDefault(LosslessJson);
 
 function fixupMetadata(metadata) {
@@ -55,6 +57,7 @@ async function avroToBuffer(params) {
             const buffers = [];
             const opts = {
                 writeHeader: true,
+                codecs: { deflate: zlib__namespace.deflateRaw },
                 codec: 'deflate',
                 metadata,
             };
@@ -120,7 +123,12 @@ function _icebergToAvroField(field, schema) {
             }
             throw new Error(`Unsupported transform: ${field.transform} for type`);
     }
-    return { name: field.name, type: ['null', avroType], default: null };
+    return {
+        name: field.name,
+        type: ['null', avroType],
+        default: null,
+        'field-id': field['field-id'],
+    };
 }
 function _mapPrimitiveToAvro(type) {
     switch (type) {
@@ -166,7 +174,9 @@ var ListContent;
     ListContent[ListContent["DELETES"] = 1] = "DELETES";
 })(ListContent || (ListContent = {}));
 const BigIntType = avsc__namespace.types.LongType.__with({
-    fromBuffer: (buf) => buf.readBigInt64LE(),
+    fromBuffer(uint_array) {
+        return Buffer.from(uint_array).readBigInt64LE();
+    },
     toBuffer(n) {
         const buf = Buffer.alloc(8);
         buf.writeBigInt64LE(n);
@@ -531,19 +541,19 @@ const ManifestListType = avsc__namespace.Type.forSchema({
             'field-id': 503,
         },
         {
-            name: 'added_data_files_count',
+            name: 'added_files_count',
             type: 'int',
             doc: 'Added entry count',
             'field-id': 504,
         },
         {
-            name: 'existing_data_files_count',
+            name: 'existing_files_count',
             type: 'int',
             doc: 'Existing entry count',
             'field-id': 505,
         },
         {
-            name: 'deleted_data_files_count',
+            name: 'deleted_files_count',
             type: 'int',
             doc: 'Deleted entry count',
             'field-id': 506,
@@ -837,10 +847,12 @@ async function updateManifestList(params) {
     }
     const passthrough = new node_stream.PassThrough();
     const decoder = new avsc__namespace.streams.BlockDecoder({
+        codecs: { deflate: zlib__namespace.inflateRaw },
         parseHook: () => ManifestListType,
     });
     const encoder = new avsc__namespace.streams.BlockEncoder(ManifestListType, {
         codec: 'deflate',
+        codecs: { deflate: zlib__namespace.deflateRaw },
         metadata,
     });
     encoder.pipe(passthrough);
@@ -961,9 +973,9 @@ async function addManifest(params) {
         sequence_number: params.sequenceNumber,
         min_sequence_number: params.sequenceNumber,
         added_snapshot_id: params.snapshotId,
-        added_data_files_count: params.files.length,
-        existing_data_files_count: 0,
-        deleted_data_files_count: 0,
+        added_files_count: params.files.length,
+        existing_files_count: 0,
+        deleted_files_count: 0,
         added_rows_count,
         existing_rows_count: 0n,
         deleted_rows_count: 0n,
@@ -1168,6 +1180,19 @@ async function addPartitionSpec(params) {
         },
     });
 }
+async function removeSnapshots(params) {
+    return icebergRequest({
+        tableBucketARN: params.tableBucketARN,
+        method: 'POST',
+        suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
+        body: {
+            requirements: [],
+            updates: [
+                { action: 'remove-snapshots', 'snapshot-ids': params.snapshotIds },
+            ],
+        },
+    });
+}
 
 const DEFAULT_RETRY_COUNT = 5;
 async function addDataFiles(params) {
@@ -1192,7 +1217,18 @@ async function addDataFiles(params) {
     if (snapshot && !old_list_key) {
         throw new Error('last snapshot invalid');
     }
-    let sequence_number = BigInt(metadata.snapshots.reduce((memo, s) => s['sequence-number'] > memo ? s['sequence-number'] : memo, 0)) + 1n;
+    let sequence_number = BigInt(metadata['last-sequence-number']) + 1n;
+    let remove_snapshot_id = 0n;
+    if (params.maxSnapshots && metadata.snapshots.length >= params.maxSnapshots) {
+        let earliest_time = 0;
+        for (const snap of metadata.snapshots) {
+            const snap_time = snap['timestamp-ms'];
+            if (earliest_time === 0 || snap_time < earliest_time) {
+                earliest_time = snap_time;
+                remove_snapshot_id = BigInt(snap['snapshot-id']);
+            }
+        }
+    }
     let added_files = 0;
     let added_records = 0n;
     let added_size = 0n;
@@ -1252,6 +1288,37 @@ async function addDataFiles(params) {
             });
         }
         try {
+            const updates = [
+                {
+                    action: 'add-snapshot',
+                    snapshot: {
+                        'sequence-number': sequence_number,
+                        'snapshot-id': snapshot_id,
+                        'parent-snapshot-id': parent_snapshot_id,
+                        'timestamp-ms': Date.now(),
+                        summary: {
+                            operation: 'append',
+                            'added-data-files': String(added_files),
+                            'added-records': String(added_records),
+                            'added-files-size': String(added_size),
+                        },
+                        'manifest-list': manifest_list_url,
+                        'schema-id': metadata['current-schema-id'],
+                    },
+                },
+                {
+                    action: 'set-snapshot-ref',
+                    'snapshot-id': snapshot_id,
+                    type: 'branch',
+                    'ref-name': 'main',
+                },
+            ];
+            if (remove_snapshot_id > 0n) {
+                updates.push({
+                    action: 'remove-snapshots',
+                    'snapshot-ids': [remove_snapshot_id],
+                });
+            }
             const result = await icebergRequest({
                 credentials: params.credentials,
                 tableBucketARN: params.tableBucketARN,
@@ -1267,31 +1334,7 @@ async function addDataFiles(params) {
                             },
                         ]
                         : [],
-                    updates: [
-                        {
-                            action: 'add-snapshot',
-                            snapshot: {
-                                'sequence-number': sequence_number,
-                                'snapshot-id': snapshot_id,
-                                'parent-snapshot-id': parent_snapshot_id,
-                                'timestamp-ms': Date.now(),
-                                summary: {
-                                    operation: 'append',
-                                    'added-data-files': String(added_files),
-                                    'added-records': String(added_records),
-                                    'added-files-size': String(added_size),
-                                },
-                                'manifest-list': manifest_list_url,
-                                'schema-id': metadata['current-schema-id'],
-                            },
-                        },
-                        {
-                            action: 'set-snapshot-ref',
-                            'snapshot-id': snapshot_id,
-                            type: 'branch',
-                            'ref-name': 'main',
-                        },
-                    ],
+                    updates,
                 },
             });
             return {
@@ -1305,7 +1348,10 @@ async function addDataFiles(params) {
         catch (e) {
             if (e instanceof IcebergHttpError &&
                 e.status === 409 &&
-                try_count < retry_max) ;
+                try_count < retry_max) {
+                // retry case
+                remove_snapshot_id = 0n;
+            }
             else {
                 throw e;
             }
@@ -1344,6 +1390,7 @@ async function setCurrentCommit(params) {
         method: 'POST',
         suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
         body: {
+            requirements: [],
             updates: [
                 {
                     action: 'set-snapshot-ref',
@@ -1374,6 +1421,7 @@ var index = {
     addManifest,
     addDataFiles,
     setCurrentCommit,
+    removeSnapshots,
 };
 
 exports.IcebergHttpError = IcebergHttpError;
@@ -1383,4 +1431,5 @@ exports.addPartitionSpec = addPartitionSpec;
 exports.addSchema = addSchema;
 exports.default = index;
 exports.getMetadata = getMetadata;
+exports.removeSnapshots = removeSnapshots;
 exports.setCurrentCommit = setCurrentCommit;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-s3tables",
-  "version": "0.0.9",
+  "version": "0.0.11",
   "description": "node api for dealing with s3tables",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -28,7 +28,7 @@
     "@aws-sdk/client-s3": "^3.901.0",
     "@aws-sdk/client-s3tables": "^3.901.0",
     "@aws-sdk/lib-storage": "^3.901.0",
-    "avsc": "^5.7.9",
+    "avsc": "https://github.com/jim-lake/avsc.git#66bc46724a6db2e55123a94532aca318a133a8ee",
     "lossless-json": "^4.2.0"
   },
   "devDependencies": {