@es-labs/jslib 0.0.1

package/services/ali.js

@@ -0,0 +1,279 @@
+// Aliyun OSS interface - https://github.com/ali-sdk/ali-oss
+// suitable for files that are not large... limit to 10Mb file size
+//
+// res.status = 204 no content but still success
+// res.status = 200
+// res.statusMessage = 200
+// TDB signatureUrlV4 & usage
+
+import 'dotenv/config.js' // node --env-file .env
+import OSS from 'ali-oss'
+
+const { OSS_AK_ID, OSS_AK_SECRET, OSS_REGION, OSS_BUCKET } = process.env
+
+const store = (OSS_AK_ID && OSS_AK_SECRET && OSS_REGION) ? new OSS({
+  region: OSS_REGION,
+  accessKeyId: OSS_AK_ID,
+  accessKeySecret: OSS_AK_SECRET,
+  bucket: OSS_BUCKET,
+}) : null
+
+/**
+ * get count of objects in a bucket.
+ * @param {{ bucketName: string }}
+ * @returns {{ status: Number, count: Number }} 
+ */
+const countBucketObjects = async (bucketName = null) => {
+  try {
+    const result = await store.getBucketStat(bucketName)
+    return { status: 200, count: result?.stat?.ObjectCount }
+  } catch (e) {
+    // status 404, code: NoSuchBucket
+    return { status: e.status, count: 0 }
+  }
+}
+
+/**
+ * A point on a two dimensional plane.
+ * @typedef {Object} ListV2Object
+ * @property {string} name - e.g. arco/a5.wav
+ * @property {string} url - e.g. http://my-bucket.oss-ap-southeast-1.aliyuncs.com/test/a5.wav
+ * @property {string} lastModified - e.g. 2024-09-03T02:34:24.000Z
+ * @property {string} etag - e.g. "3A627F876FD033F4B5CB81F063F9F883"
+ * @property {string} type - e.g. Normal
+ * @property {Number} size - e.g. 948328
+ * @property {string} storageClass - e.g. Standard
+ * @property {string} owner - e.g. null
+ */
+
+/**
+ * get count of objects in a bucket.
+ * @param {{ prefix: string, maxKeys: Number }}
+ * @returns {{ status: Number, statusMessage: string, [objects]: ListV2Object[] }} 
+ */
+const listObjects = async ({ prefix = '', maxKeys = 10 } = {}) => {
+  // console.log(prefix, maxKeys)
+  try {
+    const result = await store.listV2({
+      prefix,
+      'max-keys': maxKeys
+    })
+    // console.log(result.res)
+    const { status, statusMessage } = result.res
+    return {
+      status,
+      statusMessage,
+      objects: result.objects
+    }
+  } catch (e) {
+    console.log('ali - listObjects', e)
+    return { status: 500, statusMessage: e.toString() }
+  }
+}
+
+/**
+ * put object in a bucket.
+ * @param {string} key - the object key - e.g. test/hello.txt
+ * @param {string|Buffer|ReadableStream} payload - file data
+ * @returns {{ status: Number, statusMessage: string }} 
+ */
+const putObject = async (key, payload) => {
+  // if (!store) return null
+  try {
+    const result = await store.put(key, payload) // 
+    // console.log(result)
+    const { status, statusMessage } = result.res
+    return { status, statusMessage }
+  } catch (e) {
+    console.log('ali - putObject', e)
+    return { status: 500, statusMessage: e.toString() }
+  }
+}
+
+/**
+ * get object.
+ * @param {string} key - the object key - e.g. test/hello.txt
+ * @param {string|Buffer|ReadableStream} payload - file data
+ * @returns {{ status: Number, statusMessage: string, [buffer]: Buffer }} 
+ */
+const getObject = async (key) => {
+  // if (!store) return null
+  try {
+    const result = await store.get(key)
+    // console.log(typeof result.content, result.content.toString('utf-8')) // content is Buffer object
+    // console.log(Buffer.isBuffer(result.content))
+    // console.log(result)
+    const { status, statusMessage} = result?.res || {}
+    return {
+      status,
+      statusMessage,
+      buffer: result?.content
+    }
+  } catch (e) {
+    console.log('ali - getObject', e)
+    return { status: 500, statusMessage: e.toString() }
+  }
+}
+
+/**
+ * get object.
+ * @param {string[]} keys - the object keys - e.g. ['test/hello.txt','abc/d123.txt']
+ * @returns {{ status: Number, statusMessage: string, [deleted]: { Key: string }[] }}
+ */
+const deleteObjects = async (keys) => {
+  // if (!store) return null
+  try {
+    const result = await store.deleteMulti(keys, {})
+    // console.log(result)
+    const { status, statusMessage } = result?.res || {}
+    return {
+      status,
+      statusMessage,
+      deleted: result.deleted
+    }
+  } catch (e) {
+    console.log('ali - deleteObjects', e)
+    return { status: 500, statusMessage: e.toString() }
+  }
+}
+
+// (method, expires[, request, objectName, additionalHeaders])
+/**
+ * get signed URL.
+ * @param {string} method - GET or PUT
+ * @param {number} expires - expiration in seconds
+ * @param {string} key - the object key - e.g. test/hello.txt
+ */
+const getSignedUrl = async (method, expires, key, headers = null, additional = null) => {
+  const signedUrl = await store.signatureUrlV4(method, expires, headers, key, additional)
+  console.log(signedUrl)
+  return signedUrl
+}
+
+/**
+ * get a signed URL to write a file to OSS.
+ * @param {string} directory - the directory in OSS.
+ * @param {string} filename - the filename
+ * @param {string} contentType - the content type of the file
+ * @param {string} [action='write'] - 'write' or 'read'
+ * @param {number} [expiration=7200] - the expiration time of the signed URL
+ * @param {object} [callbackConfig] - the callback config for write action, example: { callback_url: 'https://example.com', body: { foo: 'bar' } }
+ * @returns {{ url: string, error?: string }}
+ */
+const getUploadURL = async (directory, filename, contentType, action = 'write', expiration = 7200, callbackConfig) => {
+  if (!action || !filename) {
+    return { error: 'filename and action required' }
+  }
+
+  try {
+    let url
+
+    // write / new file action
+    if (action === 'write') {
+      let arr = filename.split('.')
+
+      arr[0] = crypto
+        .createHash('sha256')
+        .update(arr[0] + new Date().getTime())
+        .digest('hex')
+
+      let newFilename = arr.join('.')
+
+      const fullPath = directory ? `${directory}/${newFilename}` : newFilename
+
+      url = await store.signatureUrl(fullPath, {
+        expires: expiration,
+        method: 'PUT',
+        'Content-Type': contentType,
+        callback: {
+          url: callbackConfig?.callback_url,
+          body: JSON.stringify({...callbackConfig?.body, filename, filepath: fullPath}),
+          contentType: 'application/json'
+        }
+      })
+    } else {
+      url = await store.signatureUrl(filename)
+    }
+
+    return { url }
+  } catch (e) {
+    return { error: e.toString() }
+  }
+}
+
+const test = async () => {
+  // [bucket count]
+  // const bucketObjCount = await countBucketObjects('no-such-bucket') // non-existing bucket, also test with existing bucket
+  // console.log('bucketObjCount', bucketObjCount) // -1 if error ?
+
+  // [put] - if put same object name will replace...
+  // const testFile1 = new File(['Hello, world 1!'], 'hello.txt', { type: 'text/plain' })
+  // const testData1 = await testFile1.arrayBuffer()
+  // const putRes1 = await putObject('hello1.txt', Buffer.from(testData1))
+  // console.log('putRes1', putRes1)
+
+  // const testFile2 = new File(['Hello, world 2!'], 'hello.txt', { type: 'text/plain' })
+  // const testData2 = await testFile2.arrayBuffer()
+  // const putRes2 = await putObject('hello2.txt', Buffer.from(testData2))
+  // console.log('putRes2', putRes2)
+
+  // [list objects]
+  // const listRes = await listObjects({ prefix: 'hello' })
+  // console.log('listRes.objects', listRes?.objects?.length, listRes?.objects?.map(item => item.name))
+
+  // [get object]
+  // const data1 = await getObject('hello1.txt')
+  // console.log('data1', data1?.buffer?.toString())
+  // const data2 = await getObject('hello2.txt')
+  // console.log('data2', data2?.buffer?.toString())
+
+  // [delete objects]
+  // const deleteRes = await deleteObjects(['ahello1.txt', 'ahello2.txt', 'ahello3.txt'])
+  // console.log(deleteRes)
+
+  const url = await getSignedUrl('GET', 60, 'hello1.txt')
+  /*
+  // -------------------------------------------------
+  //  PutObject
+  const putObejctUrl = await store.signatureUrlV4('PUT', 60, undefined, 'your obejct name');
+  console.log(putObejctUrl);
+  // --------------------------------------------------
+  const putObejctUrl = await store.signatureUrlV4(
+    'PUT',
+    60,
+    {
+      headers: {
+        'Content-Type': 'text/plain',
+        'Content-MD5': 'xxx',
+        'Content-Length': 1
+      }
+    },
+    'your obejct name',
+    ['Content-Length']
+  );
+  console.log(putObejctUrl);
+  */
+}
+
+// test()
+
+// async function listBucketInventory() {
+//   const bucket = 'Your Bucket Name'
+//   let nextContinuationToken
+//   // list all inventory of the bucket
+//   do {
+//     const result = await store.listBucketInventory(bucket, nextContinuationToken)
+//     console.log(result.inventoryList)
+//     nextContinuationToken = result.nextContinuationToken
+//   } while (nextContinuationToken)
+// }
+// listBucketInventory()
+export {
+  countBucketObjects,
+  listObjects,
+  putObject,
+  getObject,
+  deleteObjects,
+  getSignedUrl,
+  getUploadURL
+}

package/services/aws.js

@@ -0,0 +1,194 @@
+// TBD not used yet
+import {
+  S3Client,
+  S3ServiceException,
+  // This command supersedes the ListObjectsCommand and is the recommended way to list objects.
+  paginateListObjectsV2,
+} from "@aws-sdk/client-s3";
+
+/**
+ * Log all of the object keys in a bucket.
+ * @param {{ bucketName: string, pageSize: string }}
+ */
+export const main = async ({ bucketName, pageSize }) => {
+  const client = new S3Client({});
+  /** @type {string[][]} */
+  const objects = [];
+  try {
+    const paginator = paginateListObjectsV2(
+      { client, /* Max items per page */ pageSize: Number.parseInt(pageSize) },
+      { Bucket: bucketName },
+    );
+
+    for await (const page of paginator) {
+      objects.push(page.Contents.map((o) => o.Key));
+    }
+    objects.forEach((objectList, pageNum) => {
+      console.log(
+        `Page ${pageNum + 1}\n------\n${objectList.map((o) => `• ${o}`).join("\n")}\n`,
+      );
+    });
+  } catch (caught) {
+    if (
+      caught instanceof S3ServiceException &&
+      caught.name === "NoSuchBucket"
+    ) {
+      console.error(
+        `Error from S3 while listing objects for "${bucketName}". The bucket doesn't exist.`,
+      );
+    } else if (caught instanceof S3ServiceException) {
+      console.error(
+        `Error from S3 while listing objects for "${bucketName}".  ${caught.name}: ${caught.message}`,
+      );
+    } else {
+      throw caught;
+    }
+  }
+};
+
+
+
+/*
+
+// https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_Scenario_PresignedUrl_section.html
+// https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html
+// https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/javascript_s3_code_examples.html#basics
+
+import https from "https";
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { fromIni } from "@aws-sdk/credential-providers";
+import { HttpRequest } from "@smithy/protocol-http";
+import {
+  getSignedUrl,
+  S3RequestPresigner,
+} from "@aws-sdk/s3-request-presigner";
+import { parseUrl } from "@smithy/url-parser";
+import { formatUrl } from "@aws-sdk/util-format-url";
+import { Hash } from "@smithy/hash-node";
+
+const createPresignedUrlWithoutClient = async ({ region, bucket, key }) => {
+  const url = parseUrl(`https://${bucket}.s3.${region}.amazonaws.com/${key}`);
+  const presigner = new S3RequestPresigner({
+    credentials: fromIni(),
+    region,
+    sha256: Hash.bind(null, "sha256"),
+  });
+
+  const signedUrlObject = await presigner.presign(
+    new HttpRequest({ ...url, method: "PUT" }),
+  );
+  return formatUrl(signedUrlObject);
+};
+
+const createPresignedUrlWithClient = ({ region, bucket, key }) => {
+  const client = new S3Client({ region });
+  const command = new PutObjectCommand({ Bucket: bucket, Key: key });
+  return getSignedUrl(client, command, { expiresIn: 3600 });
+};
+
+function put(url, data) {
+  return new Promise((resolve, reject) => {
+    const req = https.request(
+      url,
+      { method: "PUT", headers: { "Content-Length": new Blob([data]).size } },
+      (res) => {
+        let responseBody = "";
+        res.on("data", (chunk) => {
+          responseBody += chunk;
+        });
+        res.on("end", () => {
+          resolve(responseBody);
+        });
+      },
+    );
+    req.on("error", (err) => {
+      reject(err);
+    });
+    req.write(data);
+    req.end();
+  });
+}
+
+export const main = async () => {
+  const REGION = "us-east-1";
+  const BUCKET = "example_bucket";
+  const KEY = "example_file.txt";
+
+  // There are two ways to generate a presigned URL.
+  // 1. Use createPresignedUrl without the S3 client.
+  // 2. Use getSignedUrl in conjunction with the S3 client and GetObjectCommand.
+  try {
+    const noClientUrl = await createPresignedUrlWithoutClient({
+      region: REGION,
+      bucket: BUCKET,
+      key: KEY,
+    });
+
+    const clientUrl = await createPresignedUrlWithClient({
+      region: REGION,
+      bucket: BUCKET,
+      key: KEY,
+    });
+
+    // After you get the presigned URL, you can provide your own file
+    // data. Refer to put() above.
+    console.log("Calling PUT using presigned URL without client");
+    await put(noClientUrl, "Hello World");
+
+    console.log("Calling PUT using presigned URL with client");
+    await put(clientUrl, "Hello World");
+
+    console.log("\nDone. Check your S3 console.");
+  } catch (err) {
+    console.error(err);
+  }
+};
+*/
+
+/*
+import {
+  DeleteObjectCommand,
+  S3Client,
+  S3ServiceException,
+  waitUntilObjectNotExists,
+} from "@aws-sdk/client-s3";
+
+// Delete one object from an Amazon S3 bucket.
+// @param {{ bucketName: string, key: string }}
+export const main = async ({ bucketName, key }) => {
+  const client = new S3Client({});
+
+  try {
+    await client.send(
+      new DeleteObjectCommand({
+        Bucket: bucketName,
+        Key: key,
+      }),
+    );
+    await waitUntilObjectNotExists(
+      { client },
+      { Bucket: bucketName, Key: key },
+    );
+    // A successful delete, or a delete for a non-existent object, both return
+    // a 204 response code.
+    console.log(
+      `The object "${key}" from bucket "${bucketName}" was deleted, or it didn't exist.`,
+    );
+  } catch (caught) {
+    if (
+      caught instanceof S3ServiceException &&
+      caught.name === "NoSuchBucket"
+    ) {
+      console.error(
+        `Error from S3 while deleting object from ${bucketName}. The bucket doesn't exist.`,
+      );
+    } else if (caught instanceof S3ServiceException) {
+      console.error(
+        `Error from S3 while deleting object from ${bucketName}.  ${caught.name}: ${caught.message}`,
+      );
+    } else {
+      throw caught;
+    }
+  }
+};
+*/

package/services/db/__tests__/keyv.spec.js

@@ -0,0 +1,31 @@
+const assert = require('node:assert/strict');
+const { describe, it, beforeEach, afterEach } = require('node:test');
+
+const StoreKeyV = require('../keyv');
+
+describe('StoreKeyV (keyv wrapper)', () => {
+  let store;
+
+  beforeEach(() => {
+    store = new StoreKeyV();
+    store.open();
+  });
+
+  afterEach(() => {
+    store.close();
+  });
+
+  it('should open a keyv instance and allow set/get', async () => {
+    const kv = store.get();
+    assert.ok(kv, 'keyv instance should be available');
+
+    await kv.set('test-key', 'value');
+    const got = await kv.get('test-key');
+    assert.strictEqual(got, 'value');
+  });
+
+  it('should clear internal reference on close', () => {
+    store.close();
+    assert.strictEqual(store.get(), null);
+  });
+});

package/services/db/keyv.js

@@ -0,0 +1,14 @@
+import { Keyv } from 'keyv'
+
+export default class StoreKeyV {
+	constructor(options = JSON.parse(process.env.KEYV_CACHE || null) || {}) {
+    this._KEYV_CACHE = options
+    this._keyv = null
+  }
+  open () {
+    this._keyv = this._KEYV_CACHE ? new Keyv(this._KEYV_CACHE) : new Keyv()
+    this._keyv.on('error', err => console.error('keyv Connection Error', err))
+  }
+  get () { return this._keyv }
+  close () { this._keyv = null }
+}

package/services/db/knex.js

@@ -0,0 +1,67 @@
+import Knex from 'knex'
+
+export default class StoreKnex {
+	constructor(options = JSON.parse(process.env.KNEXFILE || null) || {}) {
+    this._KNEXFILE = options
+    this._knex = null
+  }
+
+  async open() {
+    if (!this._KNEXFILE) console.log('KNEXFILE property empty or undefined - knex not started')
+    else {
+      try {
+        this._knex = Knex(this._KNEXFILE)
+        // sqlite, may need to use another statement with other sql dbs
+        await this._knex.raw('select 1+1 as result')
+        .then(() => console.log('knex CONNECTED'))
+        .catch(err => { console.log('DB error: ' + err.toString()) })
+      } catch (e) {
+        console.log('knex CONNECT ERROR', e.toString())
+      }
+    }
+  }
+  get () { return this._knex }
+  async close () {
+    if (this._knex) await this._knex.destroy()
+    console.log('knex closed')
+  }
+}
+
+// NOSONAR
+// Model.knex().destroy(() => {}) // returns a promise
+// Update with your config settings.
+// Mysql 8 issue for now
+// ALTER USER 'user'@'%' IDENTIFIED WITH mysql_native_password BY 'user123!@#PK';
+// FLUSH PRIVILEGES;
+// npx knex migrate:make create_users --env development
+// npx knex migrate:latest --env development
+// npx knex seed:make seed_name --env development
+// npx knex seed:run --env development
+// migrations
+//
+// exports.up = function (knex, Promise) {
+//   return Promise.all([
+//     knex.schema.createTable('ideas', table => {
+//       table.increments('id').primary()
+//       table.string('idea')
+//       table.string('creator')
+//     })
+//   ])
+// }
+// exports.down = function (knex, Promise) {
+//   return Promise.all([
+//     knex.schema.dropTable('ideas')
+//   ])
+// }
+// seeds
+// exports.seed = function (knex, Promise) {
+//   return knex('ideas').del().then(() => {
+//     return knex('ideas').insert([
+//         {creator: 'Ali', idea: 'A To Do List app!'},
+//         {creator: 'Ali', idea: 'A Blog!'},
+//         {creator: 'Ali', idea: 'A calculator'}
+//     ])
+//   })
+// }
+
+// https://dev.to/aspittel/objection--knex--painless-postgresql-in-your-node-app--6n6

package/services/db/redis.js

@@ -0,0 +1,51 @@
+// {
+//   port: 6379,
+//   host: '127.0.0.1',
+//   family: 4, // 4 (IPv4) or 6 (IPv6)
+//   password: 'auth',
+//   db: 0,
+//   // if using sentinels
+//   // sentinels: [{ host: 'localhost', port: 26379 }, { host: 'localhost', port: 26380 }],
+//   // name: 'mymaster',
+// }
+// var availableSlaves = [{ ip: '127.0.0.1', port: '31231', flags: 'slave' }]
+// var preferredSlaves = [ { ip: '127.0.0.1', port: '31231', prio: 1 }, { ip: '127.0.0.1', port: '31232', prio: 2 } ]
+// // preferredSlaves function format
+// preferredSlaves = function(availableSlaves) {
+//   for (var i = 0; i < availableSlaves.length; i++) {
+//     var slave = availableSlaves[i]
+//     if (slave.ip === '127.0.0.1' && slave.port === '31234') return slave
+//   }
+//   // if no preferred slaves are available a random one is used
+//   return false
+// }
+// var redis = new Redis({
+//   sentinels: [{ host: '127.0.0.1', port: 26379 }, { host: '127.0.0.1', port: 26380 }],
+//   name: 'mymaster',
+//   role: 'slave',
+//   preferredSlaves: preferredSlaves
+// })
+
+import Redis from 'ioredis'
+
+export default class StoreRedis {
+	constructor(options = JSON.parse(process.env.REDIS_CONFIG || null) || {}) {
+    this._REDIS_CONFIG = options
+    this._redis = null
+  }
+
+  open () {
+    const redisOpts = this._REDIS_CONFIG.opts
+    if (this._REDIS_CONFIG.retry) redisOpts.retryStrategy = (times) => Math.min(times * this._REDIS_CONFIG.retry.step, this._REDIS_CONFIG.retry.max)
+    if (this._REDIS_CONFIG.reconnect) redisOpts.reconnectOnError = (err) => err.message.includes(this._REDIS_CONFIG.reconnect.targetError) ? true : false
+    this._redis = new Redis(redisOpts)
+  }
+
+  get () { return this._redis }
+  close () {
+    if (this._redis) {
+      this._redis.disconnect()
+      this._redis = null  
+    }   
+  }
+}

package/services/index.js

@@ -0,0 +1,57 @@
+import StoreKeyV from './db/keyv.js'
+import StoreKnex from './db/knex.js'
+import StoreRedis from './db/redis.js'
+import Wss from './websocket.js'
+// import auth from '../auth/index.js'
+import '../auth/index.js'
+
+let servicesConfig = []
+const services = {}
+
+const start = async (
+  config = JSON.parse(process.env.SERVICES_CONFIG || null) || [], server = null, app = null
+) => {
+  const serviceTypesAvailable = process.env.SERVICES_TYPES_AVAILABLE.split(',')
+  try {
+    servicesConfig = config
+    servicesConfig.forEach(svc => {
+      const opts = JSON.parse(process.env[svc.options] || null)
+      if (opts && svc.type === 'knex' && StoreKnex) services[svc.name] = new StoreKnex(opts)
+      if (opts && svc.type === 'redis' && StoreRedis) services[svc.name] = new StoreRedis(opts)
+      if (opts && svc.type === 'keyv' && StoreKeyV) services[svc.name] = new StoreKeyV(opts)
+      if (opts && svc.type === 'ws' && Wss) services[svc.name] = new Wss(opts)
+
+      if (opts) {
+        if (svc.type === 'ws') {
+          services[svc.name].open(server, app) // set server or get app object
+        } else {
+          services[svc.name].open()
+        }  
+      }
+    })
+  } catch (e) {
+    console.log(e)
+  }
+}
+
+const stop = async () => {
+  // console.log('services - stop - begin')
+  try {
+    const promises = servicesConfig.map(svc => services[svc.name].close())
+    await Promise.allSettled(promises)
+  } catch (e) {
+    console.log(e.toString())
+  }
+  // console.log('services - stop - end')
+}
+
+const get = (service) => services[service]?.get() || null;
+
+const list = () => servicesConfig;
+
+export {
+  start,
+  stop,
+  get,
+  list
+}