node-s3tables 0.0.1

package/README.md

@@ -0,0 +1 @@
+# node-s3tables

package/dist/index.d.ts

@@ -0,0 +1,89 @@
+import { AwsCredentialIdentity } from '@aws-sdk/types';
+import { S3TablesClientConfig } from '@aws-sdk/client-s3tables';
+
+type IcebergTransform = 'identity' | 'year' | 'month' | 'day' | 'hour' | `bucket[${number}]` | `truncate[${number}]`;
+interface IcebergPartitionField {
+    'field-id': number;
+    name: string;
+    'source-id': number;
+    transform: IcebergTransform;
+}
+type IcebergPrimitiveType = 'boolean' | 'int' | 'long' | 'float' | 'double' | 'date' | 'time' | 'timestamp' | 'timestamptz' | 'string' | 'uuid' | 'binary' | `decimal(${number},${number})` | `fixed[${number}]`;
+type IcebergComplexType = {
+    type: 'list';
+    element: IcebergType;
+    'element-required': boolean;
+} | {
+    type: 'map';
+    key: IcebergType;
+    value: IcebergType;
+    'value-required': boolean;
+} | {
+    type: 'struct';
+    fields: IcebergSchemaField[];
+};
+type IcebergType = IcebergPrimitiveType | IcebergComplexType;
+interface IcebergSchemaField {
+    id: number;
+    name: string;
+    type: IcebergType;
+    required: boolean;
+    doc?: string;
+}
+interface IcebergSchema {
+    type: 'struct';
+    'schema-id': number;
+    fields: IcebergSchemaField[];
+}
+interface IcebergPartitionSpec {
+    'spec-id': number;
+    fields: IcebergPartitionField[];
+}
+interface IcebergMetadata {
+    'last-column-id': number;
+    'current-schema-id': number;
+    schemas: IcebergSchema[];
+    'default-spec-id': number;
+    'partition-specs': IcebergPartitionSpec[];
+    'last-partition-id': number;
+    'current-snapshot-id': number;
+}
+
+type TableLocation = {
+    tableArn: string;
+} | {
+    tableBucketARN: string;
+    namespace: string;
+    name: string;
+};
+type GetMetadataParams = TableLocation & {
+    config?: S3TablesClientConfig;
+};
+declare function getMetadata(params: GetMetadataParams): Promise<IcebergMetadata>;
+interface AddSchemaParams {
+    credentials?: AwsCredentialIdentity;
+    tableBucketARN: string;
+    namespace: string;
+    name: string;
+    schemaId: number;
+    fields: IcebergSchemaField[];
+}
+declare function addSchema(params: AddSchemaParams): Promise<string>;
+interface AddPartitionSpecParams {
+    credentials?: AwsCredentialIdentity;
+    tableBucketARN: string;
+    namespace: string;
+    name: string;
+    specId: number;
+    fields: IcebergPartitionField[];
+}
+declare function addPartitionSpec(params: AddPartitionSpecParams): Promise<string>;
+
+declare const _default: {
+    getMetadata: typeof getMetadata;
+    addSchema: typeof addSchema;
+    addPartitionSpec: typeof addPartitionSpec;
+};
+
+export { addPartitionSpec, addSchema, _default as default, getMetadata };
+export type { AddPartitionSpecParams, AddSchemaParams, GetMetadataParams, IcebergComplexType, IcebergMetadata, IcebergPartitionField, IcebergPartitionSpec, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergTransform, IcebergType, TableLocation };

package/dist/index.js

@@ -0,0 +1,130 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+var clientS3 = require('@aws-sdk/client-s3');
+var clientS3tables = require('@aws-sdk/client-s3tables');
+var signatureV4 = require('@smithy/signature-v4');
+var sha256Js = require('@aws-crypto/sha256-js');
+var protocolHttp = require('@smithy/protocol-http');
+var credentialProviderNode = require('@aws-sdk/credential-provider-node');
+
+async function icebergRequest(params) {
+    const region = params.tableBucketARN.split(':')[3];
+    if (!region) {
+        throw new Error('bad tableBucketARN');
+    }
+    const arn = encodeURIComponent(params.tableBucketARN);
+    const hostname = `s3tables.${region}.amazonaws.com`;
+    const full_path = `/iceberg/v1/${arn}${params.suffix}`;
+    const body = params.body ? JSON.stringify(params.body) : null;
+    const req_opts = {
+        method: params.method ?? 'GET',
+        protocol: 'https:',
+        path: full_path,
+        hostname,
+        headers: { host: hostname },
+    };
+    if (body && req_opts.headers) {
+        req_opts.body = body;
+        req_opts.headers['content-type'] = 'application/json';
+        req_opts.headers['content-length'] = String(Buffer.byteLength(body));
+    }
+    const request = new protocolHttp.HttpRequest(req_opts);
+    const signer = new signatureV4.SignatureV4({
+        credentials: params.credentials ?? credentialProviderNode.defaultProvider(),
+        region,
+        service: 's3tables',
+        sha256: sha256Js.Sha256,
+    });
+    const signed = await signer.sign(request);
+    const url = `https://${hostname}${signed.path}`;
+    const fetch_opts = {
+        method: signed.method,
+        headers: signed.headers,
+    };
+    if (signed.body) {
+        fetch_opts.body = signed.body;
+    }
+    const res = await fetch(url, fetch_opts);
+    if (!res.ok) {
+        throw new Error(`request failed: ${res.status} ${res.statusText}`);
+    }
+    return (await res.json());
+}
+
+async function getMetadata(params) {
+    const { config, ...other } = params;
+    const client = new clientS3tables.S3TablesClient(config ?? {});
+    const get_table_cmd = new clientS3tables.GetTableCommand(other);
+    const response = await client.send(get_table_cmd);
+    if (!response.metadataLocation) {
+        throw new Error('missing metadataLocation');
+    }
+    const s3_client = new clientS3.S3Client(config ?? {});
+    const { key, bucket } = _parseS3Url(response.metadataLocation);
+    const get_file_cmd = new clientS3.GetObjectCommand({ Bucket: bucket, Key: key });
+    const file_response = await s3_client.send(get_file_cmd);
+    const body = await file_response.Body?.transformToString();
+    if (!body) {
+        throw new Error('missing body');
+    }
+    return JSON.parse(body);
+}
+async function addSchema(params) {
+    return icebergRequest({
+        tableBucketARN: params.tableBucketARN,
+        method: 'POST',
+        suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
+        body: {
+            requirements: [],
+            updates: [
+                {
+                    action: 'add-schema',
+                    schema: {
+                        'schema-id': params.schemaId,
+                        type: 'struct',
+                        fields: params.fields,
+                    },
+                },
+                { action: 'set-current-schema', 'schema-id': params.schemaId },
+            ],
+        },
+    });
+}
+async function addPartitionSpec(params) {
+    return icebergRequest({
+        tableBucketARN: params.tableBucketARN,
+        method: 'POST',
+        suffix: `/namespaces/${params.namespace}/tables/${params.name}`,
+        body: {
+            requirements: [],
+            updates: [
+                {
+                    action: 'add-spec',
+                    spec: {
+                        'spec-id': params.specId,
+                        type: 'struct',
+                        fields: params.fields,
+                    },
+                },
+                { action: 'set-default-spec', 'spec-id': params.specId },
+            ],
+        },
+    });
+}
+const S3_REGEX = /^s3:\/\/([^/]+)\/(.+)$/;
+function _parseS3Url(url) {
+    const match = S3_REGEX.exec(url);
+    if (!match) {
+        throw new Error('Invalid S3 URL');
+    }
+    return { bucket: match[1], key: match[2] };
+}
+
+var index = { getMetadata, addSchema, addPartitionSpec };
+
+exports.addPartitionSpec = addPartitionSpec;
+exports.addSchema = addSchema;
+exports.default = index;
+exports.getMetadata = getMetadata;

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "node-s3tables",
+  "version": "0.0.1",
+  "description": "node api for dealing with s3tables",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist/**/*"
+  ],
+  "scripts": {
+    "build": "rollup -c",
+    "ts-check": "tsc --noEmit",
+    "lint": "eslint src test",
+    "pretty": "prettier -u --write \"**/*\" --log-level warn",
+    "test": "tsx --test test/*.test.ts",
+    "test:cover": "tsx --test --experimental-test-coverage test/*.test.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git@github.com:jim-lake/node-mysql-cron.git"
+  },
+  "author": {
+    "name": "Jim Lake"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.899.0",
+    "@aws-sdk/client-s3tables": "^3.899.0",
+    "avro-js": "^1.12.0"
+  },
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^28.0.6",
+    "@rollup/plugin-node-resolve": "^16.0.1",
+    "@rollup/plugin-typescript": "^12.1.4",
+    "@types/node": "^24.6.0",
+    "eslint": "^9.36.0",
+    "eslint-plugin-import": "^2.32.0",
+    "rollup": "^4.52.3",
+    "rollup-plugin-dts": "^6.2.3",
+    "tslib": "^2.8.1",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.2",
+    "typescript-eslint": "^8.45.0"
+  }
+}