node-s3tables 0.0.4 → 0.0.5

package/README.md

@@ -310,6 +310,82 @@ Supported partition transforms:
 - `'bucket[N]'` - Hash bucket with N buckets
 - `'truncate[N]'` - Truncate strings to N characters
 
+## AWS API Calls and Required Permissions
+
+The library makes calls to multiple AWS services and requires specific IAM permissions:
+
+### S3 Tables Service
+
+**API Calls:**
+
+- `GetTable` - Used by `getMetadata()` when called with `tableArn`
+- Iceberg REST API calls via HTTPS to `s3tables.{region}.amazonaws.com`
+
+**Required Permissions:**
+
+- `s3tables:GetTable` - For retrieving table information
+- `s3tables:GetTableData` - For reading table metadata and data objects (includes GetObject, HeadObject, ListParts)
+- `s3tables:PutTableData` - For writing table metadata and data objects (includes PutObject, multipart upload operations)
+- `s3tables:UpdateTableMetadataLocation` - For updating table root pointer during metadata operations
+
+### Function-Specific Permission Requirements
+
+**`getMetadata()`:**
+
+- When using `tableArn`: `s3tables:GetTable`, `s3tables:GetTableData`
+- When using `tableBucketARN` + `namespace` + `name`: `s3tables:GetTableData`
+
+**`addSchema()`:**
+
+- `s3tables:PutTableData`, `s3tables:UpdateTableMetadataLocation`
+
+**`addPartitionSpec()`:**
+
+- `s3tables:PutTableData`, `s3tables:UpdateTableMetadataLocation`
+
+**`addManifest()`:**
+
+- `s3tables:PutTableData` (for writing manifest files)
+
+**`addDataFiles()`:**
+
+- `s3tables:GetTableData` (to get current metadata and read existing manifest lists)
+- `s3tables:PutTableData` (to write new manifest files and lists)
+- `s3tables:UpdateTableMetadataLocation` (to add snapshots)
+
+**`setCurrentCommit()`:**
+
+- `s3tables:PutTableData`, `s3tables:UpdateTableMetadataLocation`
+
+### Example IAM Policy
+
+```json
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3tables:GetTable",
+        "s3tables:GetTableData",
+        "s3tables:PutTableData",
+        "s3tables:UpdateTableMetadataLocation"
+      ],
+      "Resource": "arn:aws:s3tables:*:*:bucket/*/table/*"
+    }
+  ]
+}
+```
+
+## Configuration
+
+The library uses the AWS SDK for authentication. Configure credentials using:
+
+- Environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`)
+- AWS credentials file (`~/.aws/credentials`)
+- IAM roles (when running on EC2/Lambda)
+- Or pass credentials directly to functions
+
 ## Testing
 
 ### Prerequisites
@@ -367,7 +443,7 @@ The test suite uses additional dependencies for creating test data:
 Run the test suite:
 
 ```bash
-npm test
+npm run test
 ```
 
 Run tests with coverage:
@@ -382,15 +458,6 @@ Run a single test file:
 npm run test:single test/create.test.ts
 ```
 
-## Configuration
-
-The library uses the AWS SDK for authentication. Configure credentials using:
-
-- Environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`)
-- AWS credentials file (`~/.aws/credentials`)
-- IAM roles (when running on EC2/Lambda)
-- Or pass credentials directly to functions
-
 ## License
 
 MIT

package/dist/index.d.ts

@@ -67,13 +67,16 @@ interface IcebergPartitionSpec {
     'spec-id': number;
     fields: IcebergPartitionField[];
 }
+interface IcebergSnapshotSummary extends Record<string, string> {
+    operation: 'append' | 'replace' | 'overwrite' | 'delete';
+}
 interface IcebergSnapshot {
     'snapshot-id': bigint | number;
     'parent-snapshot-id'?: bigint | number;
     'sequence-number': number;
     'timestamp-ms': number;
     'manifest-list': string;
-    summary: Record<string, string>;
+    summary: IcebergSnapshotSummary;
     'schema-id'?: number;
 }
 interface IcebergMetadata {
@@ -84,7 +87,7 @@ interface IcebergMetadata {
     'default-spec-id': number;
     'partition-specs': IcebergPartitionSpec[];
     'last-partition-id': number;
-    'current-snapshot-id': bigint | number;
+    'current-snapshot-id'?: bigint | number;
     location: string;
 }
 
@@ -165,8 +168,16 @@ interface AddDataFilesParams {
     namespace: string;
     name: string;
     lists: AddFileList[];
+    retryCount?: number;
+}
+interface AddDataFilesResult {
+    result: JSONObject;
+    retriesNeeded: number;
+    parentSnapshotId: bigint;
+    snapshotId: bigint;
+    sequenceNumber: bigint;
 }
-declare function addDataFiles(params: AddDataFilesParams): Promise<JSONObject>;
+declare function addDataFiles(params: AddDataFilesParams): Promise<AddDataFilesResult>;
 interface SetCurrentCommitParams {
     credentials?: AwsCredentialIdentity;
     tableBucketARN: string;
@@ -186,4 +197,4 @@ declare const _default: {
 };
 
 export { addDataFiles, addManifest, addPartitionSpec, addSchema, _default as default, getMetadata, setCurrentCommit };
-export type { AddDataFilesParams, AddFile, AddFileList, AddManifestParams, AddPartitionSpecParams, AddSchemaParams, GetMetadataParams, IcebergComplexType, IcebergMetadata, IcebergPartitionField, IcebergPartitionSpec, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSnapshot, IcebergTransform, IcebergType, SetCurrentCommitParams, TableLocation };
+export type { AddDataFilesParams, AddDataFilesResult, AddFile, AddFileList, AddManifestParams, AddPartitionSpecParams, AddSchemaParams, GetMetadataParams, IcebergComplexType, IcebergMetadata, IcebergPartitionField, IcebergPartitionSpec, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSnapshot, IcebergSnapshotSummary, IcebergTransform, IcebergType, SetCurrentCommitParams, TableLocation };

package/dist/index.js

@@ -1022,6 +1022,13 @@ function parse(text) {
     return LosslessJson__namespace.parse(text, null, customNumberParser);
 }
 
+class HttpError extends Error {
+    status;
+    constructor(status, message) {
+        super(message);
+        this.status = status;
+    }
+}
 async function icebergRequest(params) {
     const region = params.tableBucketARN.split(':')[3];
     if (!region) {
@@ -1062,8 +1069,17 @@ async function icebergRequest(params) {
     const res = await fetch(url, fetch_opts);
     const text = await res.text();
     if (!res.ok) {
-        throw new Error(`request failed: ${res.status} ${res.statusText} ${text}`);
+        if (res.status) {
+            throw new HttpError(res.status, `request failed: ${res.statusText} ${text}`);
+        }
+        throw new Error(`request failed: ${res.statusText} ${text}`);
     }
+    const ret = res.headers.get('content-type') === 'application/json'
+        ? _parse(text)
+        : text;
+    return ret;
+}
+function _parse(text) {
     try {
         return parse(text);
     }
@@ -1145,26 +1161,31 @@ async function addPartitionSpec(params) {
     });
 }
 
+const DEFAULT_RETRY_COUNT = 5;
 async function addDataFiles(params) {
     const { credentials } = params;
+    const retry_max = params.retryCount ?? DEFAULT_RETRY_COUNT;
     const region = params.tableBucketARN.split(':')[3];
     if (!region) {
         throw new Error('bad tableBucketARN');
     }
     const snapshot_id = _randomBigInt64();
     const metadata = await getMetadata(params);
-    const parent_snapshot_id = metadata['current-snapshot-id'];
     const bucket = metadata.location.split('/').slice(-1)[0];
-    const snapshot = parent_snapshot_id === -1
-        ? null
-        : metadata.snapshots.find((s) => s['snapshot-id'] === parent_snapshot_id);
+    const parent_snapshot_id = BigInt(metadata['current-snapshot-id'] ?? -1n);
+    const snapshot = metadata.snapshots.find((s) => s['snapshot-id'] === parent_snapshot_id) ??
+        null;
     if (!bucket) {
         throw new Error('bad manifest location');
     }
-    if (parent_snapshot_id !== -1 && !snapshot) {
+    if (parent_snapshot_id > 0n && !snapshot) {
         throw new Error('no old snapshot');
     }
-    const sequence_number = BigInt(metadata.snapshots.reduce((memo, s) => s['sequence-number'] > memo ? s['sequence-number'] : memo, 0)) + 1n;
+    let old_list_key = snapshot ? parseS3Url(snapshot['manifest-list']).key : '';
+    if (snapshot && !old_list_key) {
+        throw new Error('last snapshot invalid');
+    }
+    let sequence_number = BigInt(metadata.snapshots.reduce((memo, s) => s['sequence-number'] > memo ? s['sequence-number'] : memo, 0)) + 1n;
     let added_files = 0;
     let added_records = 0n;
     let added_size = 0n;
@@ -1186,88 +1207,126 @@ async function addDataFiles(params) {
         };
         return addManifest(opts);
     }));
-    const manifest_list_key = `metadata/${node_crypto.randomUUID()}.avro`;
-    const manifest_list_url = `s3://${bucket}/${manifest_list_key}`;
-    if (snapshot) {
-        const { key: old_list_key } = parseS3Url(snapshot['manifest-list']);
-        if (!old_list_key) {
-            throw new Error('snapshot invalid');
+    let expected_snapshot_id = parent_snapshot_id;
+    for (let try_count = 0;; try_count++) {
+        const manifest_list_key = `metadata/${node_crypto.randomUUID()}.avro`;
+        const manifest_list_url = `s3://${bucket}/${manifest_list_key}`;
+        if (old_list_key) {
+            await updateManifestList({
+                credentials,
+                region,
+                bucket,
+                key: old_list_key,
+                outKey: manifest_list_key,
+                metadata: {
+                    'sequence-number': String(sequence_number),
+                    'snapshot-id': String(snapshot_id),
+                    'parent-snapshot-id': String(parent_snapshot_id),
+                },
+                prepend: records,
+            });
         }
-        await updateManifestList({
-            credentials,
-            region,
-            bucket,
-            key: old_list_key,
-            outKey: manifest_list_key,
-            metadata: {
-                'sequence-number': String(sequence_number),
-                'snapshot-id': String(snapshot_id),
-                'parent-snapshot-id': String(parent_snapshot_id),
-            },
-            prepend: records,
-        });
-    }
-    else {
-        const manifest_list_buf = await avroToBuffer({
-            type: ManifestListType,
-            metadata: {
-                'sequence-number': String(sequence_number),
-                'snapshot-id': String(snapshot_id),
-                'parent-snapshot-id': String(parent_snapshot_id),
-            },
-            records,
-        });
-        await writeS3File({
-            credentials,
-            region,
-            bucket,
-            key: manifest_list_key,
-            body: manifest_list_buf,
-        });
-    }
-    const commit_result = await icebergRequest({
-        credentials: params.credentials,
-        tableBucketARN: params.tableBucketARN,
-        method: 'POST',
-        suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
-        body: {
-            requirements: parent_snapshot_id === -1
-                ? []
-                : [
-                    {
-                        type: 'assert-ref-snapshot-id',
-                        ref: 'main',
-                        'snapshot-id': parent_snapshot_id,
-                    },
-                ],
-            updates: [
-                {
-                    action: 'add-snapshot',
-                    snapshot: {
-                        'sequence-number': sequence_number,
-                        'snapshot-id': snapshot_id,
-                        'parent-snapshot-id': parent_snapshot_id,
-                        'timestamp-ms': Date.now(),
-                        summary: {
-                            operation: 'append',
-                            'added-data-files': String(added_files),
-                            'added-records': String(added_records),
-                            'added-files-size': String(added_size),
-                        },
-                        'manifest-list': manifest_list_url,
-                        'schema-id': metadata['current-schema-id'],
-                    },
+        else {
+            const manifest_list_buf = await avroToBuffer({
+                type: ManifestListType,
+                metadata: {
+                    'sequence-number': String(sequence_number),
+                    'snapshot-id': String(snapshot_id),
+                    'parent-snapshot-id': 'null',
                 },
-                {
-                    action: 'set-snapshot-ref',
-                    'snapshot-id': snapshot_id,
-                    type: 'branch',
-                    'ref-name': 'main',
+                records,
+            });
+            await writeS3File({
+                credentials,
+                region,
+                bucket,
+                key: manifest_list_key,
+                body: manifest_list_buf,
+            });
+        }
+        try {
+            const result = await icebergRequest({
+                credentials: params.credentials,
+                tableBucketARN: params.tableBucketARN,
+                method: 'POST',
+                suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
+                body: {
+                    requirements: expected_snapshot_id > 0n
+                        ? [
+                            {
+                                type: 'assert-ref-snapshot-id',
+                                ref: 'main',
+                                'snapshot-id': expected_snapshot_id,
+                            },
+                        ]
+                        : [],
+                    updates: [
+                        {
+                            action: 'add-snapshot',
+                            snapshot: {
+                                'sequence-number': sequence_number,
+                                'snapshot-id': snapshot_id,
+                                'parent-snapshot-id': parent_snapshot_id,
+                                'timestamp-ms': Date.now(),
+                                summary: {
+                                    operation: 'append',
+                                    'added-data-files': String(added_files),
+                                    'added-records': String(added_records),
+                                    'added-files-size': String(added_size),
+                                },
+                                'manifest-list': manifest_list_url,
+                                'schema-id': metadata['current-schema-id'],
+                            },
+                        },
+                        {
+                            action: 'set-snapshot-ref',
+                            'snapshot-id': snapshot_id,
+                            type: 'branch',
+                            'ref-name': 'main',
+                        },
+                    ],
                 },
-            ],
-        },
-    });
-    return commit_result;
+            });
+            return {
+                result,
+                retriesNeeded: try_count,
+                parentSnapshotId: parent_snapshot_id,
+                snapshotId: snapshot_id,
+                sequenceNumber: sequence_number,
+            };
+        }
+        catch (e) {
+            if (e instanceof HttpError && e.status === 409 && try_count < retry_max) ;
+            else {
+                throw e;
+            }
+        }
+        // we do a merge in the append only simultanious case
+        const conflict_metadata = await getMetadata(params);
+        const conflict_snapshot_id = BigInt(conflict_metadata['current-snapshot-id'] ?? -1n);
+        if (conflict_snapshot_id <= 0n) {
+            throw new Error('conflict');
+        }
+        const conflict_snap = conflict_metadata.snapshots.find((s) => s['snapshot-id'] === conflict_snapshot_id);
+        if (!conflict_snap) {
+            throw new Error('conflict');
+        }
+        if (conflict_snap.summary.operation === 'append' &&
+            BigInt(conflict_snap['sequence-number']) === sequence_number) {
+            old_list_key = parseS3Url(conflict_snap['manifest-list']).key;
+            if (!old_list_key) {
+                throw new Error('conflict');
+            }
+            added_files += parseInt(conflict_snap.summary['added-data-files'] ?? '0', 10);
+            added_records += BigInt(conflict_snap.summary['added-records'] ?? '0');
+            added_size += BigInt(conflict_snap.summary['added-files-size'] ?? '0');
+            expected_snapshot_id = conflict_snapshot_id;
+            sequence_number++;
+        }
+        else {
+            throw new Error('conflict');
+        }
+    }
 }
 async function setCurrentCommit(params) {
     const commit_result = await icebergRequest({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-s3tables",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "node api for dealing with s3tables",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",