@vrtmrz/livesync-commonlib 0.1.0-rc.0

package/dist/string_and_binary/path.js

@@ -0,0 +1,240 @@
+import { Minimatch } from "minimatch";
+import { getWebCrypto } from "../mods.js";
+import {
+  FLAGMD_REDFLAG,
+  FLAGMD_REDFLAG2,
+  FLAGMD_REDFLAG3,
+  PREFIX_OBFUSCATED,
+  PREFIXMD_LOGFILE,
+  FLAGMD_REDFLAG2_HR,
+  FLAGMD_REDFLAG3_HR,
+  PREFIXMD_LOGFILE_UC
+} from "../common/types.js";
+import { memorizeFuncWithLRUCache } from "../common/utils.js";
+import { uint8ArrayToHexString, writeString } from "./convert.js";
+import { unique } from "octagonal-wheels/collection.js";
+function isValidFilenameInWidows(filename) {
+  const regex = /[\u0000-\u001f]|[\\":?<>|*#]/g;
+  if (regex.test(filename)) return false;
+  const win = /(\\|\/)(COM\d|LPT\d|CON|PRN|AUX|NUL|CLOCK$)($|\.)/gi;
+  if (win.test(filename)) return false;
+  return true;
+}
+function isValidFilenameInDarwin(filename) {
+  const regex = /[\u0000-\u001f]|[:]/g;
+  return !regex.test(filename);
+}
+function isValidFilenameInLinux(filename) {
+  const regex = /[\u0000-\u001f]|[:]/g;
+  return !regex.test(filename);
+}
+function isValidFilenameInAndroid(filename) {
+  const regex = /[\u0000-\u001f]|[\\":?<>|*#]/g;
+  return !regex.test(filename);
+}
+function isFilePath(path) {
+  if (path.indexOf(":") === -1) return true;
+  return false;
+}
+function stripAllPrefixes(prefixedPath) {
+  if (isFilePath(prefixedPath)) return prefixedPath;
+  const [, body] = expandFilePathPrefix(prefixedPath);
+  return stripAllPrefixes(body);
+}
+function addPrefix(path, prefix) {
+  if (prefix && path.startsWith(prefix)) return path;
+  return `${prefix ?? ""}${path}`;
+}
+function expandFilePathPrefix(path) {
+  let [prefix, body] = path.split(":", 2);
+  if (!body) {
+    body = prefix;
+    prefix = "";
+  } else {
+    prefix = prefix + ":";
+  }
+  return [prefix, body];
+}
+function expandDocumentIDPrefix(id) {
+  let [prefix, body] = id.split(":", 2);
+  if (!body) {
+    body = prefix;
+    prefix = "";
+  } else {
+    prefix = prefix + ":";
+  }
+  return [prefix, body];
+}
+const _hashString = memorizeFuncWithLRUCache(async (key) => {
+  const buff = writeString(key);
+  const webcrypto = await getWebCrypto();
+  let digest = await webcrypto.subtle.digest("SHA-256", buff);
+  const len = key.length;
+  for (let i = 0; i < len; i++) {
+    digest = await webcrypto.subtle.digest("SHA-256", buff);
+  }
+  return uint8ArrayToHexString(new Uint8Array(digest));
+});
+function hashString(key) {
+  return _hashString(key);
+}
+async function path2id_base(filenameSrc, obfuscatePassphrase, caseInsensitive) {
+  if (filenameSrc.startsWith(PREFIX_OBFUSCATED)) return `${filenameSrc}`;
+  let filename = `${filenameSrc}`;
+  const newPrefix = obfuscatePassphrase ? PREFIX_OBFUSCATED : "";
+  if (caseInsensitive) {
+    filename = filename.toLowerCase();
+  }
+  let x = filename;
+  if (x.startsWith("_")) x = "/" + x;
+  if (!obfuscatePassphrase) {
+    return newPrefix + x;
+  }
+  const [prefix, body] = expandFilePathPrefix(x);
+  if (body.startsWith(PREFIX_OBFUSCATED)) return newPrefix + x;
+  const hashedPassphrase = await hashString(obfuscatePassphrase);
+  const out = await hashString(`${hashedPassphrase}:${filename}`);
+  return prefix + newPrefix + out;
+}
+function id2path_base(id, entry) {
+  if (entry && entry?.path) {
+    return id2path_base(entry.path);
+  }
+  if (id.startsWith(PREFIX_OBFUSCATED)) throw new Error("Entry has been obfuscated!");
+  const [prefix, body] = expandDocumentIDPrefix(id);
+  if (body.startsWith(PREFIX_OBFUSCATED)) throw new Error("Entry has been obfuscated!");
+  if (body.startsWith("/")) {
+    return body.substring(1);
+  }
+  return prefix + body;
+}
+function getPath(entry) {
+  return id2path_base(entry._id, entry);
+}
+function getPathWithoutPrefix(entry) {
+  const f = getPath(entry);
+  return stripAllPrefixes(f);
+}
+function stripPrefix(prefixedPath) {
+  const [prefix, body] = prefixedPath.split(":", 2);
+  if (!body) {
+    return prefix;
+  }
+  return body;
+}
+function shouldBeIgnored(filename) {
+  if (filename == FLAGMD_REDFLAG) {
+    return true;
+  }
+  if (filename == FLAGMD_REDFLAG2) {
+    return true;
+  }
+  if (filename == FLAGMD_REDFLAG2_HR) {
+    return true;
+  }
+  if (filename == FLAGMD_REDFLAG3) {
+    return true;
+  }
+  if (filename == FLAGMD_REDFLAG3_HR) {
+    return true;
+  }
+  if (filename.startsWith(PREFIXMD_LOGFILE)) {
+    return true;
+  }
+  if (filename.startsWith(PREFIXMD_LOGFILE_UC)) {
+    return true;
+  }
+  return false;
+}
+function isPlainText(filename) {
+  if (filename.endsWith(".md")) return true;
+  if (filename.endsWith(".txt")) return true;
+  if (filename.endsWith(".svg")) return true;
+  if (filename.endsWith(".html")) return true;
+  if (filename.endsWith(".csv")) return true;
+  if (filename.endsWith(".css")) return true;
+  if (filename.endsWith(".js")) return true;
+  if (filename.endsWith(".xml")) return true;
+  if (filename.endsWith(".canvas")) return true;
+  return false;
+}
+function shouldSplitAsPlainText(filename) {
+  if (filename.endsWith(".md")) return true;
+  if (filename.endsWith(".txt")) return true;
+  if (filename.endsWith(".canvas")) return true;
+  return false;
+}
+const matchOpts = { platform: "linux", dot: true, flipNegate: true, nocase: true };
+const matcherCache = /* @__PURE__ */ new WeakMap();
+function matchIgnoredPath(path, pattern, ignore) {
+  let matchers = matcherCache.get(ignore);
+  if (matchers === void 0) {
+    matchers = /* @__PURE__ */ new Map();
+    matcherCache.set(ignore, matchers);
+  }
+  let matcher = matchers.get(pattern);
+  if (matcher === void 0) {
+    matcher = new Minimatch(pattern, matchOpts);
+    matchers.set(pattern, matcher);
+  }
+  return matcher.match(path);
+}
+function isAccepted(path, ignore) {
+  if (path.indexOf("./") !== -1 || path.indexOf("../") !== -1) {
+    return false;
+  }
+  const patterns = ignore.map((e) => e.trim()).filter((e) => e.length > 0 && !e.startsWith("#"));
+  let result = void 0;
+  for (const pattern of patterns) {
+    if (pattern.endsWith("/")) {
+      if (matchIgnoredPath(path, `${pattern}**`, ignore)) {
+        return false;
+      }
+    }
+    const newResult = pattern.startsWith("!");
+    const matched = matchIgnoredPath(path, pattern, ignore) || !pattern.endsWith("/") && matchIgnoredPath(path, pattern + "/**", ignore);
+    if (matched) {
+      result = newResult;
+    }
+  }
+  return result;
+}
+async function isAcceptedAll(path, ignoreFiles, getList) {
+  const pathBase = path.substring(0, path.lastIndexOf("/"));
+  const intermediatePaths = unique(
+    pathBase.split("/").reduce((p, c) => [...p, p[p.length - 1] + "/" + c], [""]).map((e) => e.substring(1))
+  ).reverse();
+  for (const intermediatePath of intermediatePaths) {
+    for (const ignoreFile of ignoreFiles) {
+      const ignoreFilePath = intermediatePath + "/" + ignoreFile;
+      const list = await getList(ignoreFilePath);
+      if (list === false) continue;
+      const result = isAccepted(path.substring(intermediatePath.length ? intermediatePath.length + 1 : 0), list);
+      if (result !== void 0) {
+        return result;
+      }
+    }
+  }
+  return true;
+}
+export {
+  addPrefix,
+  expandDocumentIDPrefix,
+  expandFilePathPrefix,
+  getPath,
+  getPathWithoutPrefix,
+  id2path_base,
+  isAccepted,
+  isAcceptedAll,
+  isFilePath,
+  isPlainText,
+  isValidFilenameInAndroid,
+  isValidFilenameInDarwin,
+  isValidFilenameInLinux,
+  isValidFilenameInWidows,
+  path2id_base,
+  shouldBeIgnored,
+  shouldSplitAsPlainText,
+  stripAllPrefixes,
+  stripPrefix
+};

package/dist/worker/bg.common.d.ts

@@ -0,0 +1,2 @@
+import type { END_OF_DATA } from "./universalTypes.js";
+export declare function postBack(key: number, seq: number, data: string | END_OF_DATA): void;

package/dist/worker/bg.common.js

@@ -0,0 +1,6 @@
+function postBack(key, seq, data) {
+  self.postMessage({ key, seq, result: data });
+}
+export {
+  postBack
+};

package/dist/worker/bg.worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/worker/bg.worker.encryption.d.ts

@@ -0,0 +1,7 @@
+import type { EncryptHKDFArguments } from "./universalTypes.js";
+import type { EncryptArguments } from "./universalTypes.js";
+/**
+ * Processes the encryption of data.
+ * @param data The data to be encrypted or decrypted.
+ */
+export declare function processEncryption(data: EncryptArguments | EncryptHKDFArguments): Promise<void>;

package/dist/worker/bg.worker.encryption.js

@@ -0,0 +1,30 @@
+import { encrypt, decrypt } from "octagonal-wheels/encryption";
+import { encrypt as encryptHKDF, decrypt as decryptHKDF } from "octagonal-wheels/encryption/hkdf";
+async function processEncryption(data) {
+  const key = data.key;
+  const { type, input, passphrase } = data;
+  try {
+    if (type == "encrypt") {
+      const autoCalculateIterations = data.autoCalculateIterations;
+      const result = await encrypt(input, passphrase, autoCalculateIterations);
+      self.postMessage({ key, result });
+    } else if (type == "decrypt") {
+      const autoCalculateIterations = data.autoCalculateIterations;
+      const result = await decrypt(input, passphrase, autoCalculateIterations);
+      self.postMessage({ key, result });
+    } else if (type == "encryptHKDF") {
+      const pbkdf2Salt = data.pbkdf2Salt;
+      const result = await encryptHKDF(input, passphrase, pbkdf2Salt);
+      self.postMessage({ key, result });
+    } else if (type == "decryptHKDF") {
+      const pbkdf2Salt = data.pbkdf2Salt;
+      const result = await decryptHKDF(input, passphrase, pbkdf2Salt);
+      self.postMessage({ key, result });
+    }
+  } catch (ex) {
+    self.postMessage({ key, error: ex });
+  }
+}
+export {
+  processEncryption
+};

package/dist/worker/bg.worker.js

@@ -0,0 +1,14 @@
+import { processSplit } from "./bg.worker.splitting.js";
+import { processEncryption } from "./bg.worker.encryption.js";
+self.onmessage = (e) => {
+  const data = e.data.data;
+  if (data.type === "split") {
+    return processSplit(data);
+  } else if (data.type === "encrypt" || data.type === "decrypt") {
+    return processEncryption(data);
+  } else if (data.type === "encryptHKDF" || data.type === "decryptHKDF") {
+    return processEncryption(data);
+  } else {
+    self.postMessage({ key: data.key, error: new Error("Invalid type") });
+  }
+};

package/dist/worker/bg.worker.splitting.d.ts

@@ -0,0 +1,6 @@
+import type { SplitArguments } from "./universalTypes.js";
+/**
+ * Processes the splitting of data into chunks.
+ * @param data The data to be split.
+ */
+export declare function processSplit(data: SplitArguments): Promise<void>;

package/dist/worker/bg.worker.splitting.js

@@ -0,0 +1,36 @@
+import { splitPiecesRabinKarp, splitPieces2V2, splitPieces2 } from "../string_and_binary/chunks.js";
+import { postBack } from "./bg.common.js";
+import { END_OF_DATA } from "./universalTypes.js";
+function getMainThreadPostBack(key) {
+  const _key = key;
+  let seq = 0;
+  return function(data) {
+    if (data === END_OF_DATA) {
+      if (seq === 0) {
+        postBack(_key, seq++, "");
+      }
+      postBack(_key, seq++, END_OF_DATA);
+    } else {
+      postBack(_key, seq++, data);
+    }
+  };
+}
+async function processSplit(data) {
+  const key = data.key;
+  const dataSrc = data.dataSrc;
+  const pieceSize = data.pieceSize;
+  const plainSplit = data.plainSplit;
+  const minimumChunkSize = data.minimumChunkSize;
+  const filename = data.filename;
+  const useSegmenter = data.useSegmenter;
+  const func = data.splitVersion == 3 ? splitPiecesRabinKarp : data.splitVersion == 2 ? splitPieces2V2 : splitPieces2;
+  const gen = await func(dataSrc, pieceSize, plainSplit, minimumChunkSize, filename, useSegmenter);
+  const emit = getMainThreadPostBack(key);
+  for await (const v of gen()) {
+    emit(v);
+  }
+  emit(END_OF_DATA);
+}
+export {
+  processSplit
+};

package/dist/worker/bgWorker.d.ts

@@ -0,0 +1,25 @@
+import type { LiveSyncEventHub } from "../hub/hub.js";
+import type { EncryptArguments, EncryptHKDFArguments, EncryptHKDFProcessItem, EncryptProcessItem, ProcessItem, SplitArguments, SplitProcessItem } from "./universalTypes.js";
+export type WorkerInstance = {
+    worker: Worker;
+    processing: number;
+    /** Keys of tasks currently dispatched to this worker instance. */
+    taskKeys: Set<number>;
+};
+export declare function splitPieces2Worker(dataSrc: Blob, pieceSize: number, plainSplit: boolean, minimumChunkSize: number, filename?: string, useSegmenter?: boolean): () => AsyncGenerator<string, void, unknown>;
+export declare function splitPieces2WorkerV2(dataSrc: Blob, pieceSize: number, plainSplit: boolean, minimumChunkSize: number, filename?: string, useSegmenter?: boolean): () => AsyncGenerator<string, void, unknown>;
+export declare function splitPieces2WorkerRabinKarp(dataSrc: Blob, pieceSize: number, plainSplit: boolean, minimumChunkSize: number, filename?: string, useSegmenter?: boolean): () => AsyncGenerator<string, void, unknown>;
+export declare function encryptWorker(input: string, passphrase: string, autoCalculateIterations: boolean): Promise<string>;
+export declare function decryptWorker(input: string, passphrase: string, autoCalculateIterations: boolean): Promise<string>;
+export declare function encryptHKDFWorker(input: string, passphrase: string, pbkdf2Salt: Uint8Array<ArrayBuffer>): Promise<string>;
+export declare function decryptHKDFWorker(input: string, passphrase: string, pbkdf2Salt: Uint8Array<ArrayBuffer>): Promise<string>;
+export declare const tasks: Map<number, ProcessItem>;
+/**
+ * Remove a completed (or aborted) task from both the tasks map and its worker's taskKeys set.
+ */
+export declare function removeTask(key: number): void;
+export declare function initialiseWorkerModule(events: LiveSyncEventHub): void;
+export declare function startWorker(data: Omit<EncryptHKDFArguments, "key">): EncryptHKDFProcessItem;
+export declare function startWorker(data: Omit<EncryptArguments, "key">): EncryptProcessItem;
+export declare function startWorker(data: Omit<SplitArguments, "key">): SplitProcessItem;
+export declare function terminateWorker(): void;

package/dist/worker/bgWorker.direct.d.ts

@@ -0,0 +1,40 @@
+import type { EncryptHKDFProcessItem, EncryptProcessItem, SplitProcessItem, ProcessItem } from "./universalTypes.js";
+import type { LiveSyncEventHub } from "../hub/hub.js";
+export type SplitArguments = {
+    key: number;
+    type: "split";
+    dataSrc: Blob;
+    pieceSize: number;
+    plainSplit: boolean;
+    minimumChunkSize: number;
+    filename?: string;
+    useV2: boolean;
+    useSegmenter: boolean;
+};
+export type EncryptArguments = {
+    key: number;
+    type: "encrypt" | "decrypt";
+    input: string;
+    passphrase: string;
+    autoCalculateIterations: boolean;
+};
+export type EncryptHKDFArguments = {
+    key: number;
+    type: "encryptHKDF" | "decryptHKDF";
+    input: string;
+    passphrase: string;
+    pbkdf2Salt: Uint8Array;
+};
+export declare function terminateWorker(): void;
+export declare function splitPieces2Worker(dataSrc: Blob, pieceSize: number, plainSplit: boolean, minimumChunkSize: number, filename?: string, useSegmenter?: boolean): Promise<(() => Generator<string>) | (() => AsyncGenerator<string>)>;
+export declare function splitPieces2WorkerV2(dataSrc: Blob, pieceSize: number, plainSplit: boolean, minimumChunkSize: number, filename?: string, useSegmenter?: boolean): Promise<(() => Generator<string>) | (() => AsyncGenerator<string>)>;
+export declare function splitPieces2WorkerRabinKarp(dataSrc: Blob, pieceSize: number, plainSplit: boolean, minimumChunkSize: number, filename?: string, useSegmenter?: boolean): () => AsyncGenerator<string, void, unknown>;
+export declare function encryptWorker(input: string, passphrase: string, autoCalculateIterations: boolean): Promise<string>;
+export declare function decryptWorker(input: string, passphrase: string, autoCalculateIterations: boolean): Promise<string>;
+export declare function encryptHKDFWorker(input: string, passphrase: string, pbkdf2Salt: Uint8Array<ArrayBuffer>): Promise<string>;
+export declare function decryptHKDFWorker(input: string, passphrase: string, pbkdf2Salt: Uint8Array<ArrayBuffer>): Promise<string>;
+export declare function startWorker(data: Omit<EncryptHKDFArguments, "key">): EncryptHKDFProcessItem;
+export declare function startWorker(data: Omit<EncryptArguments, "key">): EncryptProcessItem;
+export declare function startWorker(data: Omit<SplitArguments, "key">): SplitProcessItem;
+export declare const tasks: Map<number, ProcessItem>;
+export declare function initialiseWorkerModule(_events: LiveSyncEventHub): void;

package/dist/worker/bgWorker.direct.js

@@ -0,0 +1,65 @@
+import { decrypt, encrypt } from "octagonal-wheels/encryption/index.js";
+import { encrypt as encryptHKDF, decrypt as decryptHKDF } from "octagonal-wheels/encryption/hkdf";
+import { splitPieces2, splitPieces2V2, splitPiecesRabinKarp } from "../string_and_binary/chunks.js";
+import { promiseWithResolvers } from "octagonal-wheels/promises";
+function terminateWorker() {
+  return;
+}
+function splitPieces2Worker(dataSrc, pieceSize, plainSplit, minimumChunkSize, filename, useSegmenter) {
+  return splitPieces2(dataSrc, pieceSize, plainSplit, minimumChunkSize, filename, useSegmenter ?? false);
+}
+function splitPieces2WorkerV2(dataSrc, pieceSize, plainSplit, minimumChunkSize, filename, useSegmenter) {
+  return splitPieces2V2(dataSrc, pieceSize, plainSplit, minimumChunkSize, filename, useSegmenter ?? false);
+}
+function splitPieces2WorkerRabinKarp(dataSrc, pieceSize, plainSplit, minimumChunkSize, filename, useSegmenter) {
+  return async function* () {
+    const splitter = await splitPiecesRabinKarp(
+      dataSrc,
+      pieceSize,
+      plainSplit,
+      minimumChunkSize,
+      filename,
+      useSegmenter ?? false
+    );
+    yield* splitter();
+  };
+}
+function encryptWorker(input, passphrase, autoCalculateIterations) {
+  return encrypt(input, passphrase, autoCalculateIterations);
+}
+function decryptWorker(input, passphrase, autoCalculateIterations) {
+  return decrypt(input, passphrase, autoCalculateIterations);
+}
+function encryptHKDFWorker(input, passphrase, pbkdf2Salt) {
+  return encryptHKDF(input, passphrase, pbkdf2Salt);
+}
+function decryptHKDFWorker(input, passphrase, pbkdf2Salt) {
+  return decryptHKDF(input, passphrase, pbkdf2Salt);
+}
+function startWorker(data) {
+  const task = promiseWithResolvers();
+  task.resolve("");
+  return {
+    task,
+    key: 0,
+    type: data.type,
+    finalize: () => {
+    }
+  };
+}
+const tasks = /* @__PURE__ */ new Map();
+function initialiseWorkerModule(_events) {
+}
+export {
+  decryptHKDFWorker,
+  decryptWorker,
+  encryptHKDFWorker,
+  encryptWorker,
+  initialiseWorkerModule,
+  splitPieces2Worker,
+  splitPieces2WorkerRabinKarp,
+  splitPieces2WorkerV2,
+  startWorker,
+  tasks,
+  terminateWorker
+};

package/dist/worker/bgWorker.encryption.d.ts

@@ -0,0 +1,22 @@
+import { type EncryptHKDFProcessItem, type ResultPayload } from "./universalTypes.js";
+import { type EncryptProcessItem } from "./universalTypes.js";
+import { type EncryptHKDFArguments } from "./universalTypes.js";
+import { type EncryptArguments } from "./universalTypes.js";
+/**
+ * Offloads encryption to a web worker.
+ * @param data The data to be encrypted.
+ * @returns A promise that resolves with the encryption result.
+ */
+export declare function encryptionOnWorker(data: Omit<EncryptArguments, "key">): Promise<string>;
+/**
+ * Offloads HKDF encryption to a web worker.
+ * @param data The data to be encrypted.
+ * @returns A promise that resolves with the encryption result.
+ */
+export declare function encryptionHKDFOnWorker(data: Omit<EncryptHKDFArguments, "key">): Promise<string>;
+/**
+ * Handles the encryption callbacks
+ * @param process The process item associated with the task.
+ * @param data The data to be processed.
+ */
+export declare function handleTaskEncrypt(process: EncryptProcessItem | EncryptHKDFProcessItem, data: ResultPayload): void;

package/dist/worker/bgWorker.encryption.js

@@ -0,0 +1,40 @@
+import { startWorker, removeTask } from "./bgWorker.js";
+import {} from "./universalTypes.js";
+import {} from "./universalTypes.js";
+import {} from "./universalTypes.js";
+import {} from "./universalTypes.js";
+function encryptionOnWorker(data) {
+  const process = startWorker(data);
+  return (async () => {
+    const ret = await process.task.promise;
+    process.finalize();
+    return ret;
+  })();
+}
+function encryptionHKDFOnWorker(data) {
+  const process = startWorker(data);
+  return (async () => {
+    const ret = await process.task.promise;
+    process.finalize();
+    return ret;
+  })();
+}
+function handleTaskEncrypt(process, data) {
+  const key = data.key;
+  const task = process.task;
+  if ("result" in data) {
+    task.resolve(data.result);
+  } else {
+    if (data.error) {
+      task.reject(data.error);
+    } else {
+      task.reject(new Error("Unknown error in background encryption"));
+    }
+  }
+  removeTask(key);
+}
+export {
+  encryptionHKDFOnWorker,
+  encryptionOnWorker,
+  handleTaskEncrypt
+};