@opentdf/sdk 0.1.0-beta.1718 → 0.2.0-beta.1941

package/src/opentdf.ts

@@ -0,0 +1,441 @@
+import { type AuthProvider } from './auth/providers.js';
+import { ConfigurationError, InvalidFileError } from './errors.js';
+import { NanoTDFDatasetClient } from './nanoclients.js';
+export { Client as TDF3Client } from '../tdf3/src/client/index.js';
+import NanoTDF from './nanotdf/NanoTDF.js';
+import decryptNanoTDF from './nanotdf/decrypt.js';
+import Client from './nanotdf/Client.js';
+import Header from './nanotdf/models/Header.js';
+import { fromSource, sourceToStream, type Source } from './seekable.js';
+import { Client as TDF3Client } from '../tdf3/src/client/index.js';
+import { AssertionConfig, AssertionVerificationKeys } from '../tdf3/src/assertions.js';
+import { type KasPublicKeyAlgorithm, OriginAllowList, isPublicKeyAlgorithm } from './access.js';
+import { type Manifest } from '../tdf3/src/models/manifest.js';
+
+export { type KasPublicKeyAlgorithm, isPublicKeyAlgorithm };
+
+export type Keys = {
+  [keyID: string]: CryptoKey | CryptoKeyPair;
+};
+
+// Options when creating a new TDF object
+// that are shared between all container types.
+export type CreateOptions = {
+  // If the policy service should be used to control creation options
+  autoconfigure?: boolean;
+
+  // List of attributes that will be assigned to the object's policy
+  attributes?: string[];
+
+  // If set and positive, this represents the maxiumum number of bytes to read from a stream to encrypt.
+  // This is helpful for enforcing size limits and preventing DoS attacks.
+  byteLimit?: number;
+
+  // The KAS to use for creation, if none is specified by the attribute service.
+  defaultKASEndpoint?: string;
+
+  // Private (or shared) keys for signing assertions and bindings
+  signers?: Keys;
+
+  // Source of plaintext data
+  source: Source;
+};
+
+export type CreateNanoTDFOptions = CreateOptions & {
+  bindingType?: 'ecdsa' | 'gmac';
+
+  // When creating a new collection, use ECDSA binding with this key id from the signers,
+  // instead of the DEK.
+  ecdsaBindingKeyID?: string;
+
+  // When creating a new collection,
+  // use the key in the `signers` list with this id
+  // to generate a signature for each element.
+  // When absent, the nanotdf is unsigned.
+  signingKeyID?: string;
+};
+
+export type CreateNanoTDFCollectionOptions = CreateNanoTDFOptions & {
+  // The maximum number of key iterations to use for a single DEK.
+  maxKeyIterations?: number;
+};
+
+// Metadata for a TDF object.
+export type Metadata = object;
+
+// MIME type of the decrypted content.
+export type MimeType = `${string}/${string}`;
+
+// Template for a Key Access Object (KAO) to be filled in during encrypt.
+export type SplitStep = {
+  // Which KAS to use to rewrap this segment of the key
+  kas: string;
+
+  // An identifier for a key segment.
+  // Leave empty to share the key.
+  sid?: string;
+};
+
+/// Options specific to the ZTDF container format.
+export type CreateZTDFOptions = CreateOptions & {
+  // Configuration for bound metadata.
+  assertionConfigs?: AssertionConfig[];
+
+  // Unbound metadata (deprecated)
+  metadata?: Metadata;
+
+  // MIME type of the decrypted content. Used for display.
+  mimeType?: MimeType;
+
+  // How to split or share the data encryption key across multiple KASes.
+  splitPlan?: SplitStep[];
+
+  // The segment size for the content; smaller is slower, but allows faster random access.
+  // The current default is 1 MiB (2^20 bytes).
+  windowSize?: number;
+
+  // Preferred algorithm to use for Key Access Objects.
+  wrappingKeyAlgorithm?: KasPublicKeyAlgorithm;
+};
+
+// Settings for decrypting any variety of TDF file.
+export type ReadOptions = {
+  // ciphertext
+  source: Source;
+  // list of KASes that may be contacted for a rewrap
+  allowedKASEndpoints?: string[];
+  // Optionally disable checking the allowlist
+  ignoreAllowlist?: boolean;
+  // Public (or shared) keys for verifying assertions
+  assertionVerificationKeys?: AssertionVerificationKeys;
+  // Optionally disable assertion verification
+  noVerify?: boolean;
+
+  // If set, prevents more than this number of concurrent requests to the KAS.
+  concurrencyLimit?: number;
+
+  // Type of key to use for wrapping responses.
+  wrappingKeyAlgorithm?: KasPublicKeyAlgorithm;
+};
+
+// Defaults and shared settings that are relevant to creating TDF objects.
+export type OpenTDFOptions = {
+  // Policy service endpoint
+  policyEndpoint?: string;
+
+  // Auth provider for connections to the policy service and KASes.
+  authProvider: AuthProvider;
+
+  // Default settings for 'encrypt' type requests.
+  defaultCreateOptions?: Omit<CreateOptions, 'source'>;
+
+  // Default settings for 'decrypt' type requests.
+  defaultReadOptions?: Omit<ReadOptions, 'source'>;
+
+  // If we want to *not* send a DPoP token
+  disableDPoP?: boolean;
+
+  // Optional keys for DPoP requests to a server.
+  // These often must be registered via a DPoP flow with the IdP
+  // which is out of the scope of this library.
+  dpopKeys?: Promise<CryptoKeyPair>;
+
+  // Configuration options for the collection header cache.
+  rewrapCacheOptions?: RewrapCacheOptions;
+};
+
+export type DecoratedStream = ReadableStream<Uint8Array> & {
+  // If the source is a TDF3/ZTDF, and includes metadata, and it has been read.
+  metadata?: Promise<unknown>;
+  manifest?: Promise<Manifest>;
+  // If the source is a NanoTDF, this will be set.
+  header?: Header;
+};
+
+// Configuration options for the collection header cache.
+export type RewrapCacheOptions = {
+  // If we should disable (bypass) the cache.
+  bypass?: boolean;
+
+  // Evict keys after this many milliseconds.
+  maxAge?: number;
+
+  // Check for expired keys once every this many milliseconds.
+  pollInterval?: number;
+};
+
+const defaultRewrapCacheOptions: Required<RewrapCacheOptions> = {
+  bypass: false,
+  maxAge: 300000,
+  pollInterval: 500,
+};
+
+// Cache for headers of nanotdf collections.
+// This allows the SDK to quickly open multiple entries of the same collection.
+// It has a demon that removes all keys that have not been accessed in the last 5 minutes.
+// To cancel the demon, and clear the cache, call `close()`.
+export class RewrapCache {
+  private cache?: Map<Uint8Array, { lastAccessTime: number; value: CryptoKey }>;
+  private closer?: ReturnType<typeof setInterval>;
+  constructor(opts?: RewrapCacheOptions) {
+    const { bypass, maxAge, pollInterval } = { ...defaultRewrapCacheOptions, ...opts };
+    if (bypass) {
+      return;
+    }
+    this.cache = new Map();
+    this.closer = setInterval(() => {
+      const now = Date.now();
+      const c = this.cache;
+      if (!c) {
+        return;
+      }
+      for (const [key, value] of c.entries()) {
+        if (now - value.lastAccessTime > maxAge) {
+          c.delete(key);
+        }
+      }
+    }, pollInterval);
+  }
+
+  get(key: Uint8Array): CryptoKey | undefined {
+    if (!this.cache) {
+      return undefined;
+    }
+    const entry = this.cache.get(key);
+    if (entry) {
+      entry.lastAccessTime = Date.now();
+      return entry.value;
+    }
+    return undefined;
+  }
+
+  set(key: Uint8Array, value: CryptoKey) {
+    if (!this.cache) {
+      return;
+    }
+    this.cache.set(key, { lastAccessTime: Date.now(), value });
+  }
+
+  close() {
+    if (this.closer !== undefined) {
+      clearInterval(this.closer);
+      delete this.closer;
+      delete this.cache;
+    }
+  }
+}
+
+// SDK for dealing with OpenTDF data and policy services.
+export class OpenTDF {
+  // Configuration service and more is at this URL/connectRPC endpoint
+  readonly policyEndpoint: string;
+  readonly authProvider: AuthProvider;
+  readonly dpopEnabled: boolean;
+  defaultCreateOptions: Omit<CreateOptions, 'source'>;
+  defaultReadOptions: Omit<ReadOptions, 'source'>;
+  readonly dpopKeys: Promise<CryptoKeyPair>;
+
+  // Header cache for reading nanotdf collections
+  private readonly rewrapCache: RewrapCache;
+  private tdf3Client: TDF3Client;
+
+  constructor({
+    authProvider,
+    dpopKeys,
+    defaultCreateOptions,
+    defaultReadOptions,
+    disableDPoP,
+    policyEndpoint,
+    rewrapCacheOptions,
+  }: OpenTDFOptions) {
+    this.authProvider = authProvider;
+    this.defaultCreateOptions = defaultCreateOptions || {};
+    this.defaultReadOptions = defaultReadOptions || {};
+    this.dpopEnabled = !!disableDPoP;
+    this.policyEndpoint = policyEndpoint || '';
+    this.rewrapCache = new RewrapCache(rewrapCacheOptions);
+    this.tdf3Client = new TDF3Client({
+      authProvider,
+      dpopKeys,
+      kasEndpoint: 'https://disallow.all.invalid',
+      policyEndpoint,
+    });
+    this.dpopKeys =
+      dpopKeys ??
+      crypto.subtle.generateKey(
+        {
+          name: 'RSASSA-PKCS1-v1_5',
+          hash: 'SHA-256',
+          modulusLength: 2048,
+          publicExponent: new Uint8Array([0x01, 0x00, 0x01]),
+        },
+        true,
+        ['sign', 'verify']
+      );
+  }
+
+  async createNanoTDF(opts: CreateNanoTDFOptions): Promise<DecoratedStream> {
+    opts = { ...this.defaultCreateOptions, ...opts };
+    const collection = await this.createNanoTDFCollection(opts);
+    try {
+      return await collection.encrypt(opts.source);
+    } finally {
+      await collection.close();
+    }
+  }
+
+  /**
+   * Creates a new collection object, which can be used to encrypt a series of data with the same policy.
+   * @returns
+   */
+  async createNanoTDFCollection(opts: CreateNanoTDFCollectionOptions): Promise<NanoTDFCollection> {
+    opts = { ...this.defaultCreateOptions, ...opts };
+    return new Collection(this.authProvider, opts);
+  }
+
+  async createZTDF(opts: CreateZTDFOptions): Promise<DecoratedStream> {
+    opts = { ...this.defaultCreateOptions, ...opts };
+    const oldStream = await this.tdf3Client.encrypt({
+      source: await sourceToStream(opts.source),
+
+      assertionConfigs: opts.assertionConfigs,
+      autoconfigure: !!opts.autoconfigure,
+      defaultKASEndpoint: opts.defaultKASEndpoint,
+      byteLimit: opts.byteLimit,
+      mimeType: opts.mimeType,
+      scope: {
+        attributes: opts.attributes,
+      },
+      splitPlan: opts.splitPlan,
+      windowSize: opts.windowSize,
+      wrappingKeyAlgorithm: opts.wrappingKeyAlgorithm,
+    });
+    const stream: DecoratedStream = oldStream.stream;
+    stream.manifest = Promise.resolve(oldStream.manifest);
+    stream.metadata = Promise.resolve(oldStream.metadata);
+    return stream;
+  }
+
+  /**
+   * Decrypts a nanotdf object. Optionally, stores the collection header and its DEK.
+   * @param ciphertext
+   */
+  async read(opts: ReadOptions): Promise<DecoratedStream> {
+    opts = { ...this.defaultReadOptions, ...opts };
+    const chunker = await fromSource(opts.source);
+    const prefix = await chunker(0, 3);
+    // switch for prefix, if starts with `PK` in ascii, or `L1L` in ascii:
+    if (prefix[0] === 0x50 && prefix[1] === 0x4b) {
+      const allowList = new OriginAllowList(opts.allowedKASEndpoints ?? [], opts.ignoreAllowlist);
+      const oldStream = await this.tdf3Client.decrypt({
+        source: opts.source,
+        allowList,
+        assertionVerificationKeys: opts.assertionVerificationKeys,
+        noVerifyAssertions: opts.noVerify,
+        wrappingKeyAlgorithm: opts.wrappingKeyAlgorithm,
+      });
+      const stream: DecoratedStream = oldStream.stream;
+      stream.metadata = Promise.resolve(oldStream.metadata);
+      return stream;
+    } else if (prefix[0] === 0x4c && prefix[1] === 0x31 && prefix[2] === 0x4c) {
+      const ciphertext = await chunker();
+      const nanotdf = NanoTDF.from(ciphertext);
+      const cachedDEK = this.rewrapCache.get(nanotdf.header.ephemeralPublicKey);
+      if (cachedDEK) {
+        const r: DecoratedStream = await streamify(decryptNanoTDF(cachedDEK, nanotdf));
+        r.header = nanotdf.header;
+        return r;
+      }
+      const nc = new Client({
+        allowedKases: opts.allowedKASEndpoints,
+        authProvider: this.authProvider,
+        ignoreAllowList: opts.ignoreAllowlist,
+        dpopEnabled: this.dpopEnabled,
+        dpopKeys: this.dpopKeys,
+        kasEndpoint: opts.allowedKASEndpoints?.[0] || 'https://disallow.all.invalid',
+      });
+      // TODO: The version number should be fetched from the API
+      const version = '0.0.1';
+      // Rewrap key on every request
+      const dek = await nc.rewrapKey(
+        nanotdf.header.toBuffer(),
+        nanotdf.header.getKasRewrapUrl(),
+        nanotdf.header.magicNumberVersion,
+        version
+      );
+      if (!dek) {
+        // These should have thrown already.
+        throw new Error('internal: key rewrap failure');
+      }
+      this.rewrapCache.set(nanotdf.header.ephemeralPublicKey, dek);
+      const r: DecoratedStream = await streamify(decryptNanoTDF(dek, nanotdf));
+      r.header = nanotdf.header;
+      return r;
+    }
+    throw new InvalidFileError(`unsupported format; prefix not recognized ${prefix}`);
+  }
+
+  close() {
+    this.rewrapCache.close();
+  }
+}
+
+async function streamify(ab: Promise<ArrayBuffer>): Promise<ReadableStream<Uint8Array>> {
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      ab.then((arrayBuffer) => {
+        controller.enqueue(new Uint8Array(arrayBuffer));
+        controller.close();
+      });
+    },
+  });
+  return stream;
+}
+
+export type NanoTDFCollection = {
+  encrypt: (source: Source) => Promise<ReadableStream<Uint8Array>>;
+  close: () => Promise<void>;
+};
+
+class Collection {
+  client?: NanoTDFDatasetClient;
+
+  constructor(authProvider: AuthProvider, opts: CreateNanoTDFCollectionOptions) {
+    if (opts.signers || opts.signingKeyID) {
+      throw new ConfigurationError('ntdf signing not implemented');
+    }
+    if (opts.autoconfigure) {
+      throw new ConfigurationError('autoconfigure not implemented');
+    }
+    if (opts.ecdsaBindingKeyID) {
+      throw new ConfigurationError('custom binding key not implemented');
+    }
+
+    this.client = new NanoTDFDatasetClient({
+      authProvider,
+      kasEndpoint: opts.defaultKASEndpoint ?? 'https://disallow.all.invalid',
+      maxKeyIterations: opts.maxKeyIterations,
+    });
+  }
+
+  async encrypt(source: Source): Promise<DecoratedStream> {
+    if (!this.client) {
+      throw new ConfigurationError('Collection is closed');
+    }
+    const chunker = await fromSource(source);
+    const cipherChunk = await this.client.encrypt(await chunker());
+    const stream: DecoratedStream = new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(new Uint8Array(cipherChunk));
+        controller.close();
+      },
+    });
+    // TODO: client's header object is private
+    // stream.header = this.client.header;
+    return stream;
+  }
+
+  async close() {
+    delete this.client;
+  }
+}

package/src/seekable.ts

@@ -0,0 +1,180 @@
+import { ConfigurationError, InvalidFileError, NetworkError } from './errors.js';
+
+/**
+ * Read data from a seekable stream.
+ * This is an abstraction for URLs with range queries and local file objects.
+ * @param byteStart First byte to read. If negative, reads from the end. If absent, reads everything
+ * @param byteEnd Index after last byte to read (exclusive)
+ */
+export type Chunker = (byteStart?: number, byteEnd?: number) => Promise<Uint8Array>;
+
+/**
+ * Type union for a variety of inputs.
+ */
+export type Source =
+  | { type: 'buffer'; location: Uint8Array }
+  | { type: 'chunker'; location: Chunker }
+  | { type: 'file-browser'; location: Blob }
+  | { type: 'remote'; location: string }
+  | { type: 'stream'; location: ReadableStream<Uint8Array> };
+
+/**
+ * Creates a seekable object from a browser file object.
+ * @param fileRef the browser file data
+ */
+export const fromBrowserFile = (fileRef: Blob): Chunker => {
+  return async (byteStart?: number, byteEnd?: number): Promise<Uint8Array> => {
+    if (byteStart === undefined) {
+      return new Uint8Array(await fileRef.arrayBuffer());
+    }
+    const chunkBlob = fileRef.slice(byteStart, byteEnd);
+    const arrayBuffer = await new Response(chunkBlob).arrayBuffer();
+    return new Uint8Array(arrayBuffer);
+  };
+};
+
+export const fromBuffer = (source: Uint8Array): Chunker => {
+  return (byteStart?: number, byteEnd?: number) => {
+    return Promise.resolve(source.slice(byteStart, byteEnd));
+  };
+};
+
+export const fromString = (source: string): Chunker => {
+  return fromBuffer(new TextEncoder().encode(source));
+};
+
+async function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function getRemoteChunk(url: string, range?: string): Promise<Uint8Array> {
+  // loop with fetch for three times, with an exponential backoff
+  // if the fetch fails with a network error
+  // this is to handle transient network errors
+  const errors: Error[] = [];
+  for (let i = 0; i < 3; i++) {
+    let res: Response;
+    try {
+      res = await fetch(url, {
+        redirect: 'follow', // manual, *follow, error
+        ...(range && {
+          headers: {
+            Range: `bytes=${range}`,
+          },
+        }),
+      });
+    } catch (e) {
+      console.warn(`fetch failed with network error [${e}], retrying...`);
+      sleep(2 ** i * 1000);
+      continue;
+    }
+    if (!res.ok) {
+      if (res.status === 416) {
+        throw new InvalidFileError(
+          `${res.status}: range not satisfiable: requested [${range}] from [${url}]; response [${res.statusText}]`
+        );
+      } else if (res.status === 404) {
+        throw new InvalidFileError(
+          `${res.status}: [${url}] not found; response: [${res.statusText}]`
+        );
+      }
+      console.warn(`fetch failed with status [${res.status}: ${res.statusText}], retrying...`);
+      // waits for 1, 2, 4 seconds
+      sleep(2 ** i * 1000);
+      continue;
+    }
+    const data = await res.arrayBuffer();
+    if (!data) {
+      throw new NetworkError(
+        `empty response for range request: requested [${range}] from [${url}]`
+      );
+    }
+    return new Uint8Array(data);
+  }
+  throw new AggregateError(errors, 'fetch failed after 3 retries');
+}
+
+export const fromUrl = async (location: string): Promise<Chunker> => {
+  return async (byteStart?: number, byteEnd?: number): Promise<Uint8Array> => {
+    if (byteStart === undefined) {
+      return getRemoteChunk(location);
+    }
+    let rangeHeader = `${byteStart}`;
+    if (byteEnd && byteEnd < 0) {
+      // NOTE: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range
+      throw Error('negative end unsupported');
+    } else if (byteEnd) {
+      rangeHeader += `-${byteEnd - 1}`;
+    }
+    return await getRemoteChunk(location, rangeHeader);
+  };
+};
+
+export const fromSource = async ({ type, location }: Source): Promise<Chunker> => {
+  switch (type) {
+    case 'buffer':
+      if (!(location instanceof Uint8Array)) {
+        throw new ConfigurationError('Invalid data source; must be uint8 array');
+      }
+      return fromBuffer(location);
+    case 'chunker':
+      if (!(location instanceof Function)) {
+        throw new ConfigurationError('Invalid data source; must be uint8 array');
+      }
+      return location;
+    case 'file-browser':
+      if (!(location instanceof Blob)) {
+        throw new ConfigurationError('Invalid data source; must be at least a Blob');
+      }
+      return fromBrowserFile(location);
+    case 'remote':
+      if (typeof location !== 'string') {
+        throw new ConfigurationError('Invalid data source; url not provided');
+      }
+      return fromUrl(location);
+    case 'stream':
+      return fromBuffer(new Uint8Array(await new Response(location).arrayBuffer()));
+    default:
+      throw new ConfigurationError(`Data source type not defined, or not supported: ${type}}`);
+  }
+};
+
+export async function sourceToStream(source: Source): Promise<ReadableStream<Uint8Array>> {
+  switch (source.type) {
+    case 'stream':
+      return source.location;
+    case 'file-browser':
+      return source.location.stream();
+    case 'chunker': {
+      const chunkSize = 8 * 1024 * 1024; // 8 megabytes
+      let offset = 0;
+      return new ReadableStream({
+        async pull(controller) {
+          const chunk = await source.location(offset, offset + chunkSize);
+          if (chunk.length === 0) {
+            controller.close();
+            return;
+          }
+          controller.enqueue(chunk);
+          offset += chunk.length;
+        },
+      });
+    }
+    default: {
+      const chunker = await fromSource(source);
+      return new ReadableStream({
+        async start(controller) {
+          const chunk = await chunker();
+          controller.enqueue(chunk);
+          controller.close();
+        },
+      });
+    }
+  }
+}
+
+// Deprected name, prefer `fromSource`
+export const fromDataSource = fromSource;
+
+// Deprecated Name; prefer just `Source`
+export type DataSource = Source;

package/src/tdf/AttributeObject.ts

@@ -7,8 +7,6 @@ export interface AttributeObject {
   /** PEM encoded public key */
   readonly pubKey: string;
   readonly kasUrl: string;
-  /** The most recent version 1.1.0. */
-  readonly schemaVersion?: string;
 }
 
 export async function createAttribute(
@@ -22,6 +20,5 @@ export async function createAttribute(
     displayName: '',
     pubKey: pubKey.publicKey,
     kasUrl,
-    schemaVersion: '1.1.0',
   };
 }

package/src/tdf/Policy.ts

@@ -1,13 +1,12 @@
 import { type AttributeObject } from './AttributeObject.js';
 import { v4 as uuid } from 'uuid';
 
-export default class Policy {
+export class Policy {
   static CURRENT_VERSION = '1.1.0';
 
   private uuidStr = uuid();
   private dataAttributesList: AttributeObject[] = [];
   private dissemList: string[] = [];
-  // private schemaVersionStr = Policy.CURRENT_VERSION;
 
   /**
    * Adds a group of entities, to the Policy's dissem list

package/src/tdf/PolicyObject.ts

@@ -5,8 +5,7 @@ export interface PolicyObjectBody {
   readonly dissem: string[];
 }
 
-export default interface PolicyObject {
+export interface PolicyObject {
   readonly uuid: string;
   readonly body: PolicyObjectBody;
-  readonly schemaVersion?: string;
 }

package/src/tdf/TypedArray.ts

@@ -1,4 +1,4 @@
-type TypedArray =
+export type TypedArray =
   | Int8Array
   | Uint8Array
   | Int16Array
@@ -8,5 +8,3 @@ type TypedArray =
   | Uint8ClampedArray
   | Float32Array
   | Float64Array;
-
-export default TypedArray;