node-s3tables 0.0.8 → 0.0.10

package/README.md

@@ -18,6 +18,7 @@ import {
   addManifest,
   addDataFiles,
   setCurrentCommit,
+  removeSnapshots,
 } from 'node-s3tables';
 
 // Get table metadata
@@ -184,6 +185,7 @@ Adds data files to an S3 table by creating a new snapshot.
 - `params.namespace` (string) - The namespace name
 - `params.name` (string) - The table name
 - `params.lists` (AddFileList[]) - Array of file lists to add
+- `params.maxSnapshots` (number, optional) - Maximum number of snapshots to retain. When set, automatically removes the oldest snapshot if the count exceeds this limit
 - `params.credentials` (AwsCredentialIdentity, optional) - AWS credentials
 
 **Returns:** Promise<string>
@@ -208,6 +210,28 @@ await addDataFiles({
     },
   ],
 });
+
+// With automatic snapshot cleanup
+await addDataFiles({
+  tableBucketARN: 'arn:aws:s3tables:us-west-2:123456789012:bucket/my-bucket',
+  namespace: 'sales',
+  name: 'daily_sales',
+  maxSnapshots: 10, // Keep only the 10 most recent snapshots
+  lists: [
+    {
+      specId: 1,
+      schemaId: 2,
+      files: [
+        {
+          file: 's3://my-bucket/data/sales-2024-01-02.parquet',
+          partitions: { sale_date_day: '2024-01-02' },
+          recordCount: 1500n,
+          fileSize: 78643n,
+        },
+      ],
+    },
+  ],
+});
 ```
 
 ### setCurrentCommit(params)
@@ -233,6 +257,29 @@ await setCurrentCommit({
 });
 ```
 
+### removeSnapshots(params)
+
+Removes snapshots from an S3 table. Note: Due to Iceberg limitations, only one snapshot can be removed at a time.
+
+**Parameters:**
+
+- `params.tableBucketARN` (string) - The ARN of the table bucket
+- `params.namespace` (string) - The namespace name
+- `params.name` (string) - The table name
+- `params.snapshotIds` (bigint[]) - Array of snapshot IDs to remove (only provide one snapshot ID)
+- `params.credentials` (AwsCredentialIdentity, optional) - AWS credentials
+
+**Returns:** Promise<IcebergUpdateResponse>
+
+```javascript
+await removeSnapshots({
+  tableBucketARN: 'arn:aws:s3tables:us-west-2:123456789012:bucket/my-bucket',
+  namespace: 'sales',
+  name: 'daily_sales',
+  snapshotIds: [4183020680887155442n], // Only one snapshot ID
+});
+```
+
 ## Type Definitions
 
 ### IcebergUpdateResponse

package/dist/index.d.ts

@@ -81,6 +81,7 @@ interface IcebergSnapshot {
 }
 interface IcebergMetadata {
     'last-column-id': number;
+    'last-sequence-number': bigint | number;
     'current-schema-id': number;
     schemas: IcebergSchema[];
     snapshots: IcebergSnapshot[];
@@ -153,6 +154,14 @@ interface AddPartitionSpecParams {
     fields: IcebergPartitionField[];
 }
 declare function addPartitionSpec(params: AddPartitionSpecParams): Promise<IcebergUpdateResponse>;
+interface RemoveSnapshotsParams {
+    credentials?: AwsCredentialIdentity;
+    tableBucketARN: string;
+    namespace: string;
+    name: string;
+    snapshotIds: bigint[];
+}
+declare function removeSnapshots(params: RemoveSnapshotsParams): Promise<IcebergUpdateResponse>;
 
 type JSONPrimitive = string | number | boolean | null | bigint | undefined;
 type JSONValue = JSONPrimitive | JSONObject | JSONArray;
@@ -174,6 +183,7 @@ interface AddDataFilesParams {
     snapshotId?: bigint;
     lists: AddFileList[];
     retryCount?: number;
+    maxSnapshots?: number;
 }
 interface AddDataFilesResult {
     result: JSONObject;
@@ -207,7 +217,8 @@ declare const _default: {
     addManifest: typeof addManifest;
     addDataFiles: typeof addDataFiles;
     setCurrentCommit: typeof setCurrentCommit;
+    removeSnapshots: typeof removeSnapshots;
 };
 
-export { IcebergHttpError, addDataFiles, addManifest, addPartitionSpec, addSchema, _default as default, getMetadata, setCurrentCommit };
-export type { AddDataFilesParams, AddDataFilesResult, AddFile, AddFileList, AddManifestParams, AddPartitionSpecParams, AddSchemaParams, GetMetadataParams, IcebergComplexType, IcebergMetadata, IcebergPartitionField, IcebergPartitionSpec, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSnapshot, IcebergSnapshotSummary, IcebergTransform, IcebergType, IcebergUpdateResponse, SetCurrentCommitParams, TableLocation };
+export { IcebergHttpError, addDataFiles, addManifest, addPartitionSpec, addSchema, _default as default, getMetadata, removeSnapshots, setCurrentCommit };
+export type { AddDataFilesParams, AddDataFilesResult, AddFile, AddFileList, AddManifestParams, AddPartitionSpecParams, AddSchemaParams, GetMetadataParams, IcebergComplexType, IcebergMetadata, IcebergPartitionField, IcebergPartitionSpec, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSnapshot, IcebergSnapshotSummary, IcebergTransform, IcebergType, IcebergUpdateResponse, RemoveSnapshotsParams, SetCurrentCommitParams, TableLocation };

package/dist/index.js

@@ -745,16 +745,16 @@ function _encodeValue(raw, transform, out_type) {
     }
 }
 const NaNValue = NaN;
-function makeBounds(paritions, spec, schema) {
+function makeBounds(partitions, spec, schema) {
     return spec.fields.map((f) => {
         const schemaField = schema.fields.find((sf) => sf.id === f['source-id']);
         if (!schemaField) {
             throw new Error(`Schema field not found for source-id ${f['source-id']}`);
         }
-        if (!(f.name in paritions)) {
-            throw new Error(`paritions missing ${f.name}`);
+        if (!(f.name in partitions)) {
+            throw new Error(`partitions missing ${f.name}`);
         }
-        const raw = paritions[f.name];
+        const raw = partitions[f.name];
         if (typeof raw === 'number' && isNaN(raw)) {
             return NaNValue;
         }
@@ -1168,6 +1168,19 @@ async function addPartitionSpec(params) {
         },
     });
 }
+async function removeSnapshots(params) {
+    return icebergRequest({
+        tableBucketARN: params.tableBucketARN,
+        method: 'POST',
+        suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
+        body: {
+            requirements: [],
+            updates: [
+                { action: 'remove-snapshots', 'snapshot-ids': params.snapshotIds },
+            ],
+        },
+    });
+}
 
 const DEFAULT_RETRY_COUNT = 5;
 async function addDataFiles(params) {
@@ -1181,8 +1194,7 @@ async function addDataFiles(params) {
     const metadata = await getMetadata(params);
     const bucket = metadata.location.split('/').slice(-1)[0];
     const parent_snapshot_id = BigInt(metadata['current-snapshot-id']);
-    const snapshot = metadata.snapshots.find((s) => s['snapshot-id'] === parent_snapshot_id) ??
-        null;
+    const snapshot = metadata.snapshots.find((s) => BigInt(s['snapshot-id']) === parent_snapshot_id) ?? null;
     if (!bucket) {
         throw new Error('bad manifest location');
     }
@@ -1193,7 +1205,18 @@ async function addDataFiles(params) {
     if (snapshot && !old_list_key) {
         throw new Error('last snapshot invalid');
     }
-    let sequence_number = BigInt(metadata.snapshots.reduce((memo, s) => s['sequence-number'] > memo ? s['sequence-number'] : memo, 0)) + 1n;
+    let sequence_number = BigInt(metadata['last-sequence-number']) + 1n;
+    let remove_snapshot_id = 0n;
+    if (params.maxSnapshots && metadata.snapshots.length >= params.maxSnapshots) {
+        let earliest_time = 0;
+        for (const snap of metadata.snapshots) {
+            const snap_time = snap['timestamp-ms'];
+            if (earliest_time === 0 || snap_time < earliest_time) {
+                earliest_time = snap_time;
+                remove_snapshot_id = BigInt(snap['snapshot-id']);
+            }
+        }
+    }
     let added_files = 0;
     let added_records = 0n;
     let added_size = 0n;
@@ -1253,6 +1276,37 @@ async function addDataFiles(params) {
             });
         }
         try {
+            const updates = [
+                {
+                    action: 'add-snapshot',
+                    snapshot: {
+                        'sequence-number': sequence_number,
+                        'snapshot-id': snapshot_id,
+                        'parent-snapshot-id': parent_snapshot_id,
+                        'timestamp-ms': Date.now(),
+                        summary: {
+                            operation: 'append',
+                            'added-data-files': String(added_files),
+                            'added-records': String(added_records),
+                            'added-files-size': String(added_size),
+                        },
+                        'manifest-list': manifest_list_url,
+                        'schema-id': metadata['current-schema-id'],
+                    },
+                },
+                {
+                    action: 'set-snapshot-ref',
+                    'snapshot-id': snapshot_id,
+                    type: 'branch',
+                    'ref-name': 'main',
+                },
+            ];
+            if (remove_snapshot_id > 0n) {
+                updates.push({
+                    action: 'remove-snapshots',
+                    'snapshot-ids': [remove_snapshot_id],
+                });
+            }
             const result = await icebergRequest({
                 credentials: params.credentials,
                 tableBucketARN: params.tableBucketARN,
@@ -1268,31 +1322,7 @@ async function addDataFiles(params) {
                             },
                         ]
                         : [],
-                    updates: [
-                        {
-                            action: 'add-snapshot',
-                            snapshot: {
-                                'sequence-number': sequence_number,
-                                'snapshot-id': snapshot_id,
-                                'parent-snapshot-id': parent_snapshot_id,
-                                'timestamp-ms': Date.now(),
-                                summary: {
-                                    operation: 'append',
-                                    'added-data-files': String(added_files),
-                                    'added-records': String(added_records),
-                                    'added-files-size': String(added_size),
-                                },
-                                'manifest-list': manifest_list_url,
-                                'schema-id': metadata['current-schema-id'],
-                            },
-                        },
-                        {
-                            action: 'set-snapshot-ref',
-                            'snapshot-id': snapshot_id,
-                            type: 'branch',
-                            'ref-name': 'main',
-                        },
-                    ],
+                    updates,
                 },
             });
             return {
@@ -1306,7 +1336,10 @@ async function addDataFiles(params) {
         catch (e) {
             if (e instanceof IcebergHttpError &&
                 e.status === 409 &&
-                try_count < retry_max) ;
+                try_count < retry_max) {
+                // retry case
+                remove_snapshot_id = 0n;
+            }
             else {
                 throw e;
             }
@@ -1345,6 +1378,7 @@ async function setCurrentCommit(params) {
         method: 'POST',
         suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
         body: {
+            requirements: [],
             updates: [
                 {
                     action: 'set-snapshot-ref',
@@ -1375,6 +1409,7 @@ var index = {
     addManifest,
     addDataFiles,
     setCurrentCommit,
+    removeSnapshots,
 };
 
 exports.IcebergHttpError = IcebergHttpError;
@@ -1384,4 +1419,5 @@ exports.addPartitionSpec = addPartitionSpec;
 exports.addSchema = addSchema;
 exports.default = index;
 exports.getMetadata = getMetadata;
+exports.removeSnapshots = removeSnapshots;
 exports.setCurrentCommit = setCurrentCommit;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-s3tables",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "description": "node api for dealing with s3tables",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",