@boxyhq/saml-jackson 0.2.3-beta.207 → 0.2.3-beta.219

package/src/db/db.ts

@@ -0,0 +1,100 @@
+import {
+  DatabaseDriver,
+  DatabaseOption,
+  Encrypted,
+  EncryptionKey,
+  Index,
+  Storable,
+} from 'saml-jackson';
+import * as encrypter from './encrypter';
+import mem from './mem';
+import mongo from './mongo';
+import redis from './redis';
+import sql from './sql/sql';
+import store from './store';
+
+const decrypt = (res: Encrypted, encryptionKey: EncryptionKey): any => {
+  if (res.iv && res.tag) {
+    return JSON.parse(
+      encrypter.decrypt(res.value, res.iv, res.tag, encryptionKey)
+    );
+  }
+
+  return JSON.parse(res.value);
+};
+
+class DB implements DatabaseDriver {
+  private db: any;
+  private encryptionKey: EncryptionKey;
+
+  constructor(db: any, encryptionKey: EncryptionKey) {
+    this.db = db;
+    this.encryptionKey = encryptionKey;
+  }
+
+  async get(namespace: string, key: string): Promise<any> {
+    const res = await this.db.get(namespace, key);
+
+    if (!res) {
+      return null;
+    }
+
+    return decrypt(res, this.encryptionKey);
+  }
+
+  async getByIndex(namespace: string, idx: Index): Promise<any> {
+    const res = await this.db.getByIndex(namespace, idx);
+    const encryptionKey = this.encryptionKey;
+    return res.map((r) => {
+      return decrypt(r, encryptionKey);
+    });
+  }
+
+  // ttl is in seconds
+  async put(
+    namespace: string,
+    key: string,
+    val: any,
+    ttl: number = 0,
+    ...indexes: any[]
+  ): Promise<any> {
+    if (ttl > 0 && indexes && indexes.length > 0) {
+      throw new Error('secondary indexes not allow on a store with ttl');
+    }
+
+    const dbVal = this.encryptionKey
+      ? encrypter.encrypt(JSON.stringify(val), this.encryptionKey)
+      : { value: JSON.stringify(val) };
+
+    return await this.db.put(namespace, key, dbVal, ttl, ...indexes);
+  }
+
+  async delete(namespace: string, key: string): Promise<any> {
+    return await this.db.delete(namespace, key);
+  }
+
+  store(namespace: string, ttl: number = 0): Storable {
+    return store.new(namespace, this, ttl);
+  }
+}
+
+export = {
+  new: async (options: DatabaseOption) => {
+    const encryptionKey = options.encryptionKey
+      ? Buffer.from(options.encryptionKey, 'latin1')
+      : null;
+
+    switch (options.engine) {
+      case 'redis':
+        return new DB(await redis.new(options), encryptionKey);
+      case 'sql':
+        return new DB(await sql.new(options), encryptionKey);
+      case 'mongo':
+        return new DB(await mongo.new(options), encryptionKey);
+      case 'mem':
+        return new DB(await mem.new(options), encryptionKey);
+      default:
+        throw new Error('unsupported db engine: ' + options.engine);
+    }
+  },
+};

package/src/db/encrypter.ts

@@ -0,0 +1,38 @@
+import crypto from 'crypto';
+import { Encrypted, EncryptionKey } from 'saml-jackson';
+
+const ALGO = 'aes-256-gcm';
+const BLOCK_SIZE = 16; // 128 bit
+
+export const encrypt = (text: string, key: EncryptionKey): Encrypted => {
+  const iv = crypto.randomBytes(BLOCK_SIZE);
+  const cipher = crypto.createCipheriv(ALGO, key, iv);
+
+  let ciphertext = cipher.update(text, 'utf8', 'base64');
+  ciphertext += cipher.final('base64');
+
+  return {
+    iv: iv.toString('base64'),
+    tag: cipher.getAuthTag().toString('base64'),
+    value: ciphertext,
+  };
+};
+
+export const decrypt = (
+  ciphertext: string,
+  iv: string,
+  tag: string,
+  key: EncryptionKey
+): string => {
+  const decipher = crypto.createDecipheriv(
+    ALGO,
+    key,
+    Buffer.from(iv, 'base64')
+  );
+  decipher.setAuthTag(Buffer.from(tag, 'base64'));
+
+  let cleartext = decipher.update(ciphertext, 'base64', 'utf8');
+  cleartext += decipher.final('utf8');
+
+  return cleartext;
+};

package/src/db/mem.ts

@@ -0,0 +1,128 @@
+// This is an in-memory implementation to be used with testing and prototyping only
+
+import { DatabaseDriver, DatabaseOption, Index } from 'saml-jackson';
+import * as dbutils from './utils';
+
+class Mem implements DatabaseDriver {
+  private options: DatabaseOption;
+  private store: any;
+  private indexes: any;
+  private cleanup: any;
+  private ttlStore: any;
+  private ttlCleanup: any;
+  private timerId: any;
+
+  constructor(options: DatabaseOption) {
+    this.options = options;
+  }
+
+  async init(): Promise<Mem> {
+    this.store = {}; // map of key, value
+    this.indexes = {}; // map of key, Set
+    this.cleanup = {}; // map of indexes for cleanup when store key is deleted
+    this.ttlStore = {}; // map of key to ttl
+
+    if (this.options.ttl) {
+      this.ttlCleanup = async () => {
+        const now = Date.now();
+        for (const k in this.ttlStore) {
+          if (this.ttlStore[k].expiresAt < now) {
+            await this.delete(this.ttlStore[k].namespace, this.ttlStore[k].key);
+          }
+        }
+
+        if (this.options.ttl) {
+          this.timerId = setTimeout(this.ttlCleanup, this.options.ttl * 1000);
+        }
+      };
+
+      this.timerId = setTimeout(this.ttlCleanup, this.options.ttl * 1000);
+    }
+
+    return this;
+  }
+
+  async get(namespace: string, key: string): Promise<any> {
+    let res = this.store[dbutils.key(namespace, key)];
+    if (res) {
+      return res;
+    }
+
+    return null;
+  }
+
+  async getByIndex(namespace: string, idx: Index): Promise<any> {
+    const dbKeys = await this.indexes[dbutils.keyForIndex(namespace, idx)];
+
+    const ret: string[] = [];
+    for (const dbKey of dbKeys || []) {
+      ret.push(await this.get(namespace, dbKey));
+    }
+
+    return ret;
+  }
+
+  async put(
+    namespace: string,
+    key: string,
+    val: string,
+    ttl: number = 0,
+    ...indexes: any[]
+  ): Promise<any> {
+    const k = dbutils.key(namespace, key);
+
+    this.store[k] = val;
+
+    if (ttl) {
+      this.ttlStore[k] = {
+        namespace,
+        key,
+        expiresAt: Date.now() + ttl * 1000,
+      };
+    }
+
+    // no ttl support for secondary indexes
+    for (const idx of indexes || []) {
+      const idxKey = dbutils.keyForIndex(namespace, idx);
+      let set = this.indexes[idxKey];
+      if (!set) {
+        set = new Set();
+        this.indexes[idxKey] = set;
+      }
+
+      set.add(key);
+
+      const cleanupKey = dbutils.keyFromParts(dbutils.indexPrefix, k);
+      let cleanup = this.cleanup[cleanupKey];
+      if (!cleanup) {
+        cleanup = new Set();
+        this.cleanup[cleanupKey] = cleanup;
+      }
+
+      cleanup.add(idxKey);
+    }
+  }
+
+  async delete(namespace: string, key: string): Promise<any> {
+    const k = dbutils.key(namespace, key);
+
+    delete this.store[k];
+
+    const idxKey = dbutils.keyFromParts(dbutils.indexPrefix, k);
+    // delete secondary indexes and then the mapping of the seconary indexes
+    const dbKeys = this.cleanup[idxKey];
+
+    for (const dbKey of dbKeys || []) {
+      this.indexes[dbKey] && this.indexes[dbKey].delete(key);
+    }
+
+    delete this.cleanup[idxKey];
+    delete this.ttlStore[k];
+  }
+}
+
+export default {
+  new: async (options: DatabaseOption) => {
+    return await new Mem(options).init();
+  },
+};

package/src/db/mongo.ts

@@ -0,0 +1,110 @@
+import { MongoClient } from 'mongodb';
+import { DatabaseDriver, DatabaseOption, Index } from 'saml-jackson';
+import * as dbutils from './utils';
+
+type Document = {
+  value: string;
+  expiresAt: Date;
+  indexes: string[];
+};
+
+class Mongo implements DatabaseDriver {
+  private options: DatabaseOption;
+  private client!: MongoClient;
+  private collection: any;
+  private db: any;
+
+  constructor(options: DatabaseOption) {
+    this.options = options;
+  }
+
+  async init(): Promise<Mongo> {
+    this.client = new MongoClient(this.options.url);
+
+    await this.client.connect();
+
+    this.db = this.client.db();
+    this.collection = this.db.collection('jacksonStore');
+
+    await this.collection.createIndex({ indexes: 1 });
+    await this.collection.createIndex(
+      { expiresAt: 1 },
+      { expireAfterSeconds: 1 }
+    );
+
+    return this;
+  }
+
+  async get(namespace: string, key: string): Promise<any> {
+    let res = await this.collection.findOne({
+      _id: dbutils.key(namespace, key),
+    });
+    if (res && res.value) {
+      return res.value;
+    }
+
+    return null;
+  }
+
+  async getByIndex(namespace: string, idx: Index): Promise<any> {
+    const docs = await this.collection
+      .find({
+        indexes: dbutils.keyForIndex(namespace, idx),
+      })
+      .toArray();
+
+    const ret: string[] = [];
+    for (const doc of docs || []) {
+      ret.push(doc.value);
+    }
+
+    return ret;
+  }
+
+  async put(
+    namespace: string,
+    key: string,
+    val: string,
+    ttl: number = 0,
+    ...indexes: any[]
+  ): Promise<void> {
+    const doc = <Document>{
+      value: val,
+    };
+
+    if (ttl) {
+      doc.expiresAt = new Date(Date.now() + ttl * 1000);
+    }
+
+    // no ttl support for secondary indexes
+    for (const idx of indexes || []) {
+      const idxKey = dbutils.keyForIndex(namespace, idx);
+
+      if (!doc.indexes) {
+        doc.indexes = [];
+      }
+
+      doc.indexes.push(idxKey);
+    }
+
+    await this.collection.updateOne(
+      { _id: dbutils.key(namespace, key) },
+      {
+        $set: doc,
+      },
+      { upsert: true }
+    );
+  }
+
+  async delete(namespace: string, key: string): Promise<any> {
+    return await this.collection.deleteOne({
+      _id: dbutils.key(namespace, key),
+    });
+  }
+}
+
+export default {
+  new: async (options: DatabaseOption): Promise<Mongo> => {
+    return await new Mongo(options).init();
+  },
+};

package/src/db/redis.ts

@@ -0,0 +1,103 @@
+import * as redis from 'redis';
+import { DatabaseDriver, DatabaseOption, Index } from 'saml-jackson';
+import * as dbutils from './utils';
+
+class Redis implements DatabaseDriver {
+  private options: DatabaseOption;
+  private client!: any;
+
+  constructor(options: DatabaseOption) {
+    this.options = options;
+  }
+
+  async init(): Promise<Redis> {
+    let opts = {};
+
+    if (this.options && this.options.url) {
+      opts['socket'] = {
+        url: this.options.url,
+      };
+    }
+
+    this.client = redis.createClient(opts);
+    this.client.on('error', (err: any) =>
+      console.log('Redis Client Error', err)
+    );
+
+    await this.client.connect();
+
+    return this;
+  }
+
+  async get(namespace: string, key: string): Promise<any> {
+    let res = await this.client.get(dbutils.key(namespace, key));
+    if (res) {
+      return JSON.parse(res);
+    }
+
+    return null;
+  }
+
+  async getByIndex(namespace: string, idx: Index): Promise<any> {
+    const dbKeys = await this.client.sMembers(
+      dbutils.keyForIndex(namespace, idx)
+    );
+
+    const ret: string[] = [];
+    for (const dbKey of dbKeys || []) {
+      ret.push(await this.get(namespace, dbKey));
+    }
+
+    return ret;
+  }
+
+  async put(
+    namespace: string,
+    key: string,
+    val: string,
+    ttl: number = 0,
+    ...indexes: any[]
+  ): Promise<void> {
+    let tx = this.client.multi();
+    const k = dbutils.key(namespace, key);
+
+    tx = tx.set(k, JSON.stringify(val));
+
+    if (ttl) {
+      tx = tx.expire(k, ttl);
+    }
+
+    // no ttl support for secondary indexes
+    for (const idx of indexes || []) {
+      const idxKey = dbutils.keyForIndex(namespace, idx);
+      tx = tx.sAdd(idxKey, key);
+      tx = tx.sAdd(dbutils.keyFromParts(dbutils.indexPrefix, k), idxKey);
+    }
+
+    await tx.exec();
+  }
+
+  async delete(namespace: string, key: string): Promise<any> {
+    let tx = this.client.multi();
+    const k = dbutils.key(namespace, key);
+    tx = tx.del(k);
+
+    const idxKey = dbutils.keyFromParts(dbutils.indexPrefix, k);
+    // delete secondary indexes and then the mapping of the seconary indexes
+    const dbKeys = await this.client.sMembers(idxKey);
+
+    for (const dbKey of dbKeys || []) {
+      tx.sRem(dbKey, key);
+    }
+
+    tx.del(idxKey);
+
+    return await tx.exec();
+  }
+}
+
+export default {
+  new: async (options: DatabaseOption): Promise<Redis> => {
+    return await new Redis(options).init();
+  },
+};

package/src/db/sql/entity/JacksonIndex.ts

@@ -0,0 +1,44 @@
+import { EntitySchema } from 'typeorm';
+import { JacksonIndex } from '../model/JacksonIndex';
+import { JacksonStore } from '../model/JacksonStore';
+
+export default new EntitySchema({
+  name: 'JacksonIndex',
+  target: JacksonIndex,
+  columns: {
+    id: {
+      primary: true,
+      generated: true,
+      type: 'int',
+    },
+    key: {
+      type: 'varchar',
+      length: 1500,
+    },
+    storeKey: {
+      type: 'varchar',
+      length: 1500,
+    },
+  },
+  relations: {
+    // TODO: Remove the below line to see the error
+    // @ts-ignore
+    store: {
+      target: () => JacksonStore,
+      type: 'many-to-one',
+      inverseSide: 'indexes',
+      eager: true,
+      onDelete: 'CASCADE',
+    },
+  },
+  indices: [
+    {
+      name: '_jackson_index_key',
+      columns: ['key'],
+    },
+    {
+      name: '_jackson_index_key_store',
+      columns: ['key', 'storeKey'],
+    },
+  ],
+});

package/src/db/sql/entity/JacksonStore.ts

@@ -0,0 +1,43 @@
+import { DatabaseType } from 'saml-jackson';
+import { ColumnType, EntitySchema } from 'typeorm';
+import { JacksonStore } from '../model/JacksonStore';
+
+const valueType = (type: DatabaseType): ColumnType => {
+  switch (type) {
+    case 'postgres':
+    case 'cockroachdb':
+      return 'text';
+    case 'mysql':
+    case 'mariadb':
+      return 'mediumtext';
+    default:
+      return 'varchar';
+  }
+};
+
+export default (type: DatabaseType) => {
+  return new EntitySchema({
+    name: 'JacksonStore',
+    target: JacksonStore,
+    columns: {
+      key: {
+        primary: true,
+        type: 'varchar',
+        length: 1500,
+      },
+      value: {
+        type: valueType(type),
+      },
+      iv: {
+        type: 'varchar',
+        length: 64,
+        nullable: true,
+      },
+      tag: {
+        type: 'varchar',
+        length: 64,
+        nullable: true,
+      },
+    },
+  });
+};

package/src/db/sql/entity/JacksonTTL.ts

@@ -0,0 +1,17 @@
+import { Entity, Column, Index } from 'typeorm';
+
+@Entity()
+export class JacksonTTL {
+  @Column({
+    primary: true,
+    type: 'varchar',
+    length: 1500,
+  })
+  key!: string;
+
+  @Index('_jackson_ttl_expires_at')
+  @Column({
+    type: 'bigint',
+  })
+  expiresAt!: number;
+}

package/src/db/sql/model/JacksonIndex.ts

@@ -0,0 +1,3 @@
+export class JacksonIndex {
+  constructor(public id: number, public key: string, public store: any) {}
+}

package/src/db/sql/model/JacksonStore.ts

@@ -0,0 +1,8 @@
+export class JacksonStore {
+  constructor(
+    public key: string,
+    public value: any,
+    public iv: any,
+    public tag: any
+  ) {}
+}