@mblaney/holster 1.0.7 → 1.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mblaney/holster",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
   "type": "module",
   "main": "index.js",

package/src/array.js

@@ -5,29 +5,40 @@ if (typeof btoa === "undefined") {
 
 // This is Array extended to have .toString(["utf8"|"hex"|"base64"])
 function SeaArray() {}
-
 Object.assign(SeaArray, {from: Array.from})
 SeaArray.prototype = Object.create(Array.prototype)
+
 SeaArray.prototype.toString = function (enc, start, end) {
   if (!enc) enc = "utf8"
   if (!start) start = 0
 
-  const length = this.length
+  end = end ? Math.min(Math.max(end, start), this.length) : this.length
+
   if (enc === "hex") {
-    const buf = new Uint8Array(this)
-    return [...Array(((end && end + 1) || length) - start).keys()]
-      .map(i => buf[i + start].toString(16).padStart(2, "0"))
-      .join("")
+    const buf = new Uint8Array(this.slice(start, end))
+    return Array.from(buf, byte => byte.toString(16).padStart(2, "0")).join("")
   }
 
   if (enc === "utf8") {
-    return Array.from({length: (end || length) - start}, (_, i) =>
-      String.fromCharCode(this[i + start]),
-    ).join("")
+    return Array.from({length: end - start}, (_, i) => {
+      const charCode = this[i + start]
+      // Use unicode replacement character for invalid character codes.
+      if (charCode < 0 || charCode > 0x10ffff) {
+        return String.fromCharCode(0xfffd)
+      }
+      return String.fromCharCode(charCode)
+    }).join("")
   }
 
   if (enc === "base64") {
-    return btoa(this)
+    const utf8String = Array.from({length: end - start}, (_, i) => {
+      const charCode = this[i + start]
+      if (charCode < 0 || charCode > 255) {
+        return String.fromCharCode(0xfffd)
+      }
+      return String.fromCharCode(charCode)
+    }).join("")
+    return btoa(utf8String)
   }
 }
 

package/src/buffer.js

@@ -1,83 +1,416 @@
 import SeaArray from "./array.js"
 
-// This is Buffer implementation used in SEA. Functionality is mostly
-// compatible with NodeJS "safe-buffer" and is used for encoding conversions
-// between binary and "hex" | "utf8" | "base64"
-// See documentation and validation for safe implementation in:
-// https://github.com/feross/safe-buffer#update
+// Security constants
+const SECURITY_LIMITS = {
+  MAX_BUFFER_SIZE: 10 * 1024 * 1024, // 10MB limit
+  MAX_STRING_LENGTH: 1024 * 1024, // 1MB string limit
+}
+
+// Secure Buffer implementation for SEA.js
+// Compatible with NodeJS safe-buffer but with enhanced security
 function SafeBuffer(...props) {
   console.warn("new SafeBuffer() is deprecated, please use SafeBuffer.from()")
   return SafeBuffer.from(...props)
 }
 
 SafeBuffer.prototype = Object.create(Array.prototype)
-Object.assign(SafeBuffer, {
-  // (data, enc) where typeof data === "string" then
-  // enc === "utf8"|"hex"|"base64"
-  from() {
-    if (!Object.keys(arguments).length || arguments[0] == null) {
+
+// Input validation helpers
+const validateStringInput = (input, encoding) => {
+  if (typeof input !== "string") {
+    throw new TypeError("Input must be a string for encoding: " + encoding)
+  }
+
+  if (input.length > SECURITY_LIMITS.MAX_STRING_LENGTH) {
+    throw new RangeError(
+      `String too large: ${input.length} > ${SECURITY_LIMITS.MAX_STRING_LENGTH}`,
+    )
+  }
+}
+
+const validateBufferSize = (size, operation = "buffer operation") => {
+  if (size > SECURITY_LIMITS.MAX_BUFFER_SIZE) {
+    throw new RangeError(
+      `Buffer size too large for ${operation}: ${size} > ${SECURITY_LIMITS.MAX_BUFFER_SIZE}`,
+    )
+  }
+}
+
+// Secure hex parsing
+const parseHexString = hexString => {
+  validateStringInput(hexString, "hex")
+
+  // Remove any whitespace and convert to lowercase
+  const cleanHex = hexString.replace(/\s/g, "").toLowerCase()
+
+  // Validate hex format: only 0-9, a-f characters
+  if (!/^[0-9a-f]*$/.test(cleanHex)) {
+    throw new TypeError("Invalid hex string: contains non-hex characters")
+  }
+
+  // Must be even length
+  if (cleanHex.length % 2 !== 0) {
+    throw new TypeError("Invalid hex string: must have even length")
+  }
+
+  if (cleanHex.length === 0) {
+    return new Uint8Array(0)
+  }
+
+  validateBufferSize(cleanHex.length / 2, "hex parsing")
+
+  const bytes = new Uint8Array(cleanHex.length / 2)
+  for (let i = 0; i < cleanHex.length; i += 2) {
+    const hexByte = cleanHex.substr(i, 2)
+    bytes[i / 2] = parseInt(hexByte, 16)
+  }
+  return bytes
+}
+
+const encodeUtf8String = str => {
+  validateStringInput(str, "utf8")
+
+  try {
+    // Use standard TextEncoder for proper UTF-8 encoding
+    const encoder = new TextEncoder()
+    const bytes = encoder.encode(str)
+
+    validateBufferSize(bytes.length, "UTF-8 encoding")
+    return bytes
+  } catch (err) {
+    throw new TypeError("Failed to encode UTF-8 string: " + err.message)
+  }
+}
+
+const encodeBinaryString = str => {
+  validateStringInput(str, "binary")
+
+  validateBufferSize(str.length, "binary encoding")
+
+  const bytes = new Uint8Array(str.length)
+  for (let i = 0; i < str.length; i++) {
+    const charCode = str.charCodeAt(i)
+
+    // Binary strings should only contain bytes (0-255)
+    if (charCode > 255) {
       throw new TypeError(
-        "First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.",
+        `Invalid binary string: character at position ${i} exceeds byte range (${charCode} > 255)`,
       )
     }
+
+    bytes[i] = charCode
+  }
+  return bytes
+}
+
+// Secure base64 decoding
+const parseBase64String = base64String => {
+  validateStringInput(base64String, "base64")
+
+  // Remove any whitespace
+  const cleanBase64 = base64String.replace(/\s/g, "")
+
+  // Validate base64 format
+  if (!/^[A-Za-z0-9+/]*={0,2}$/.test(cleanBase64)) {
+    throw new TypeError("Invalid base64 string: contains invalid characters")
+  }
+
+  try {
+    const decodedString = atob(cleanBase64)
+    validateBufferSize(decodedString.length, "base64 decoding")
+
+    const bytes = new Uint8Array(decodedString.length)
+    for (let i = 0; i < decodedString.length; i++) {
+      bytes[i] = decodedString.charCodeAt(i)
+    }
+
+    return bytes
+  } catch (err) {
+    throw new TypeError("Failed to decode base64 string: " + err.message)
+  }
+}
+
+// Secure array-like input processing
+const processArrayLikeInput = input => {
+  let bytes
+
+  if (input instanceof ArrayBuffer) {
+    validateBufferSize(input.byteLength, "ArrayBuffer processing")
+    bytes = new Uint8Array(input)
+  } else if (ArrayBuffer.isView(input)) {
+    validateBufferSize(
+      input.byteLength || input.length,
+      "TypedArray processing",
+    )
+    bytes = new Uint8Array(input.buffer, input.byteOffset, input.byteLength)
+  } else if (Array.isArray(input)) {
+    validateBufferSize(input.length, "Array processing")
+
+    // Validate all array elements are valid byte values
+    for (let i = 0; i < input.length; i++) {
+      const val = input[i]
+      if (
+        typeof val !== "number" ||
+        val < 0 ||
+        val > 255 ||
+        !Number.isInteger(val)
+      ) {
+        throw new TypeError(
+          `Invalid byte value at index ${i}: ${val} (must be integer 0-255)`,
+        )
+      }
+    }
+
+    bytes = new Uint8Array(input)
+  } else if (
+    typeof input === "object" &&
+    input !== null &&
+    typeof input.length === "number"
+  ) {
+    // Handle array-like objects
+    validateBufferSize(input.length, "array-like processing")
+
+    const arr = Array.from(input)
+    for (let i = 0; i < arr.length; i++) {
+      const val = arr[i]
+      if (
+        typeof val !== "number" ||
+        val < 0 ||
+        val > 255 ||
+        !Number.isInteger(val)
+      ) {
+        throw new TypeError(
+          `Invalid byte value at index ${i}: ${val} (must be integer 0-255)`,
+        )
+      }
+    }
+
+    bytes = new Uint8Array(arr)
+  } else {
+    throw new TypeError("Unsupported input type: " + typeof input)
+  }
+
+  return bytes
+}
+
+Object.assign(SafeBuffer, {
+  from() {
+    const argCount = arguments.length
+
+    if (argCount === 0) {
+      throw new TypeError("SafeBuffer.from() requires at least one argument")
+    }
+
     const input = arguments[0]
-    let buf
+
+    if (input === null || input === undefined) {
+      throw new TypeError("First argument must not be null or undefined")
+    }
+
+    let bytes
+
     if (typeof input === "string") {
-      const enc = arguments[1] || "utf8"
-      if (enc === "hex") {
-        const bytes = input
-          .match(/([\da-fA-F]{2})/g)
-          .map(byte => parseInt(byte, 16))
-        if (!bytes || !bytes.length) {
-          throw new TypeError("Invalid first argument for type 'hex'.")
-        }
-        buf = SeaArray.from(bytes)
-      } else if (enc === "utf8" || enc === "binary") {
-        const length = input.length
-        const words = new Uint16Array(length)
-        Array.from({length: length}, (_, i) => (words[i] = input.charCodeAt(i)))
-        buf = SeaArray.from(words)
-      } else if (enc === "base64") {
-        const dec = atob(input)
-        const length = dec.length
-        const bytes = new Uint8Array(length)
-        Array.from({length: length}, (_, i) => (bytes[i] = dec.charCodeAt(i)))
-        buf = SeaArray.from(bytes)
-      } else {
-        console.info("SafeBuffer.from unknown encoding: " + enc)
+      const encoding = (arguments[1] || "utf8").toLowerCase()
+
+      switch (encoding) {
+        case "hex":
+          bytes = parseHexString(input)
+          break
+
+        case "utf8":
+        case "utf-8":
+          bytes = encodeUtf8String(input)
+          break
+
+        case "binary":
+        case "latin1":
+          bytes = encodeBinaryString(input)
+          break
+
+        case "base64":
+          bytes = parseBase64String(input)
+          break
+
+        case "ascii":
+          // ASCII is subset of UTF-8, but validate range
+          validateStringInput(input, "ascii")
+          for (let i = 0; i < input.length; i++) {
+            if (input.charCodeAt(i) > 127) {
+              throw new TypeError(
+                `Invalid ASCII character at position ${i}: ${input.charCodeAt(i)} > 127`,
+              )
+            }
+          }
+          bytes = encodeUtf8String(input) // Safe since ASCII is UTF-8 subset
+          break
+
+        default:
+          throw new TypeError("Unknown encoding: " + encoding)
       }
-      return buf
+    } else {
+      bytes = processArrayLikeInput(input)
     }
-    const length = input.byteLength ? input.byteLength : input.length
-    if (length) {
-      let buf
-      if (input instanceof ArrayBuffer) {
-        buf = new Uint8Array(input)
-      }
-      const check = SeaArray.from(buf || input)
-      return check
+
+    if (!bytes || bytes.length === 0) {
+      return SeaArray.from(new Uint8Array(0))
+    }
+
+    try {
+      return SeaArray.from(bytes)
+    } catch (err) {
+      throw new Error("Failed to create SafeBuffer: " + err.message)
     }
   },
-  // This is "safe-buffer.alloc" sans encoding support
+
   alloc(length, fill = 0) {
-    return SeaArray.from(
-      new Uint8Array(Array.from({length: length}, () => fill)),
-    )
+    if (typeof length !== "number" || !Number.isInteger(length) || length < 0) {
+      throw new TypeError("Length must be a non-negative integer")
+    }
+
+    validateBufferSize(length, "buffer allocation")
+
+    // Validate fill value
+    if (typeof fill === "number") {
+      if (fill < 0 || fill > 255 || !Number.isInteger(fill)) {
+        throw new TypeError("Fill value must be an integer between 0 and 255")
+      }
+    } else if (typeof fill === "string") {
+      if (fill.length !== 1) {
+        throw new TypeError("Fill string must be exactly one character")
+      }
+      fill = fill.charCodeAt(0)
+      if (fill > 255) {
+        throw new TypeError("Fill character code must be <= 255")
+      }
+    } else {
+      throw new TypeError("Fill must be a number or single character string")
+    }
+
+    try {
+      const buffer = new Uint8Array(length)
+      if (fill !== 0) {
+        buffer.fill(fill)
+      }
+      return SeaArray.from(buffer)
+    } catch (err) {
+      throw new Error("Failed to allocate buffer: " + err.message)
+    }
   },
-  // This puts together array of array like members
+
   concat(arr) {
-    // octet array
     if (!Array.isArray(arr)) {
-      throw new TypeError(
-        "First argument must be Array containing ArrayBuffer or Uint8Array instances.",
-      )
+      throw new TypeError("First argument must be an array")
+    }
+
+    if (arr.length === 0) {
+      return SeaArray.from(new Uint8Array(0))
+    }
+
+    // Validate all array elements and calculate total size
+    let totalLength = 0
+    const validatedBuffers = []
+
+    for (let i = 0; i < arr.length; i++) {
+      const item = arr[i]
+
+      if (!item) {
+        throw new TypeError(`Array element at index ${i} is null or undefined`)
+      }
+
+      let bytes
+      try {
+        bytes = processArrayLikeInput(item)
+      } catch (err) {
+        throw new TypeError(
+          `Invalid array element at index ${i}: ${err.message}`,
+        )
+      }
+
+      totalLength += bytes.length
+      validateBufferSize(totalLength, "buffer concatenation")
+
+      validatedBuffers.push(bytes)
     }
-    return SeaArray.from(
-      arr.reduce((ret, item) => ret.concat(Array.from(item)), []),
+
+    try {
+      const result = new Uint8Array(totalLength)
+      let offset = 0
+
+      for (const buffer of validatedBuffers) {
+        result.set(buffer, offset)
+        offset += buffer.length
+      }
+
+      return SeaArray.from(result)
+    } catch (err) {
+      throw new Error("Failed to concatenate buffers: " + err.message)
+    }
+  },
+
+  // Helper method to check if object is SafeBuffer
+  isBuffer(obj) {
+    return (
+      obj instanceof SeaArray ||
+      (obj && typeof obj === "object" && obj.constructor === SafeBuffer)
     )
   },
+
+  // Get byte length of string in specific encoding
+  byteLength(string, encoding = "utf8") {
+    if (typeof string !== "string") {
+      throw new TypeError("First argument must be a string")
+    }
+
+    switch (encoding.toLowerCase()) {
+      case "utf8":
+      case "utf-8":
+        return new TextEncoder().encode(string).length
+
+      case "ascii":
+      case "binary":
+      case "latin1":
+        return string.length
+
+      case "base64":
+        const clean = string.replace(/\s/g, "")
+        return (
+          Math.floor((clean.length * 3) / 4) - (clean.match(/=/g) || []).length
+        )
+
+      case "hex":
+        return Math.floor(string.replace(/\s/g, "").length / 2)
+
+      default:
+        throw new TypeError("Unknown encoding: " + encoding)
+    }
+  },
 })
+
 SafeBuffer.prototype.from = SafeBuffer.from
-SafeBuffer.prototype.toString = SeaArray.prototype.toString
+SafeBuffer.prototype.toString = function (
+  encoding = "utf8",
+  start = 0,
+  end = this.length,
+) {
+  // Input validation
+  if (typeof encoding !== "string") {
+    throw new TypeError("Encoding must be a string")
+  }
+
+  if (typeof start !== "number" || !Number.isInteger(start) || start < 0) {
+    throw new TypeError("Start must be a non-negative integer")
+  }
+
+  if (typeof end !== "number" || !Number.isInteger(end) || end < start) {
+    throw new TypeError("End must be an integer >= start")
+  }
+
+  // Delegate to SeaArray with validation
+  try {
+    return SeaArray.prototype.toString.call(this, encoding, start, end)
+  } catch (err) {
+    throw new Error("Failed to convert buffer to string: " + err.message)
+  }
+}
 
 export default SafeBuffer

package/src/dup.js

@@ -8,6 +8,8 @@ const Dup = maxAge => {
     dup.store[id] = Date.now()
     if (!dup.expiry) {
       dup.expiry = setTimeout(() => {
+        if (dup.expiry) return
+
         const now = Date.now()
         Object.keys(dup.store).forEach(id => {
           if (now - dup.store[id] > maxAge) delete dup.store[id]

package/src/get.js

@@ -1,7 +1,18 @@
 import {match} from "./utils.js"
 
 const Get = (lex, graph) => {
+  if (!lex || typeof lex !== "object") {
+    throw new TypeError("lex must be an object")
+  }
+  if (!graph || typeof graph !== "object") {
+    throw new TypeError("graph must be an object")
+  }
+
   const soul = lex["#"]
+  if (!soul || typeof soul !== "string") {
+    throw new TypeError("soul must be a string")
+  }
+
   if (!graph[soul]) return
 
   const node = {_: {"#": soul, ">": {}}}

package/src/ham.js

@@ -1,6 +1,10 @@
 import * as utils from "./utils.js"
 import SEA from "./sea.js"
 
+const LISTENER_DELAY = 100
+// Maximum number of souls to keep in memory
+const MAX_GRAPH_SIZE = 10000
+
 // state and value are the incoming changes.
 // currentState and currentValue are the current graph data.
 const Ham = (state, currentState, value, currentValue) => {
@@ -26,6 +30,16 @@ const Ham = (state, currentState, value, currentValue) => {
 }
 
 Ham.mix = async (change, graph, secure, listen) => {
+  if (!change || typeof change !== "object") {
+    throw new TypeError("change must be an object")
+  }
+  if (!graph || typeof graph !== "object") {
+    throw new TypeError("graph must be an object")
+  }
+  if (!listen || typeof listen !== "object") {
+    throw new TypeError("listen must be an object")
+  }
+
   const machine = Date.now()
   const now = {}
   const defer = {}
@@ -103,9 +117,6 @@ Ham.mix = async (change, graph, secure, listen) => {
           if (!now[soul]) {
             now[soul] = {_: {"#": soul, ">": {}}}
           }
-          // TODO: graph should not just grow indefintitely in memory.
-          // Need to have a max size after which start dropping the oldest state
-          // Do something similar to Dup which can handle deletes?
           if (!graph[soul]) {
             graph[soul] = {_: {"#": soul, ">": {}}}
           }
@@ -121,7 +132,7 @@ Ham.mix = async (change, graph, secure, listen) => {
                   .filter(l => utils.match(l["."], key))
                   .forEach(l => l.cb())
               }
-            }, 100)
+            }, LISTENER_DELAY)
           }
           updated = true
         }
@@ -139,9 +150,24 @@ Ham.mix = async (change, graph, secure, listen) => {
         if (listen[soul]) {
           listen[soul].filter(l => utils.match(l["."])).forEach(l => l.cb())
         }
-      }, 100)
+      }, LISTENER_DELAY)
     }
   }
+
+  const souls = Object.keys(graph)
+  if (souls.length > MAX_GRAPH_SIZE) {
+    // Sort by oldest state timestamp and remove oldest entries
+    const soulsByAge = souls
+      .map(soul => {
+        const states = graph[soul]._ && graph[soul]._[">"]
+        const maxState = states ? Math.max(...Object.values(states)) : 0
+        return {soul, maxState}
+      })
+      .sort((a, b) => a.maxState - b.maxState)
+    const remove = soulsByAge.slice(0, souls.length - maxSize)
+    remove.forEach(({soul}) => delete graph[soul])
+  }
+
   return {now: now, defer: defer, wait: wait}
 }
 

package/src/store.js

@@ -49,6 +49,12 @@ const fileSystem = opt => {
       },
       list: cb => {
         fs.readdir(dir, (err, files) => {
+          if (err) {
+            console.log("fs.readdir error:", err)
+            cb()
+            return
+          }
+
           files.forEach(cb)
           cb()
         })
@@ -206,10 +212,14 @@ const Store = opt => {
 
   return {
     get: (lex, cb) => {
-      if (!lex) {
+      if (!lex || !utils.obj.is(lex)) {
         cb("lex required")
         return
       }
+      if (!lex["#"]) {
+        cb("soul required in lex")
+        return
+      }
 
       var soul = lex["#"]
       var key = typeof lex["."] === "string" ? lex["."] : ""
@@ -241,18 +251,22 @@ const Store = opt => {
         return
       }
 
-      var count = 0
+      let count = 0
+      let finished = false
       const ack = err => {
         count--
-        if (ack.err) return
+        if (finished) return
 
-        ack.err = err
-        if (ack.err) {
-          cb(ack.err)
+        if (err) {
+          finished = true
+          cb(err)
           return
         }
 
-        if (count === 0) cb(null)
+        if (count === 0) {
+          finished = true
+          cb(null)
+        }
       }
 
       Object.keys(graph).forEach(soul => {