api-ape 2.2.3 → 2.3.0

package/README.md

@@ -116,6 +116,57 @@ api.onConnectionChange((state) => {
 * **HTTP streaming fallback** — Automatically falls back to long polling when WebSockets are blocked
 * **JJS Encoding** — Extended JSON supporting Date, RegExp, Error, Set, Map, undefined, and circular refs
 * **Connection lifecycle hooks** — Customize behavior on connect, receive, send, error, and disconnect
+* **🌲 Forest** — Distributed mesh for horizontal scaling across multiple servers
+
+---
+
+## 🌲 Forest: Distributed Mesh
+
+**Forest** enables horizontal scaling by coordinating multiple api-ape servers through a shared database. Messages are routed directly to the server hosting the destination client — no broadcast spam.
+
+```js
+import { createClient } from 'redis';
+const redis = createClient();
+await redis.connect();
+
+// Join the mesh — that's it!
+ape.joinVia(redis);
+```
+
+### Supported Backends
+
+| Backend | Push Mechanism | Best For |
+|---------|---------------|----------|
+| **Redis** | PUB/SUB | Most deployments |
+| **MongoDB** | Change Streams | Mongo-native stacks |
+| **PostgreSQL** | LISTEN/NOTIFY | SQL shops |
+| **Supabase** | Realtime | Supabase users |
+| **Firebase** | Native push | Serverless/edge |
+
+### How It Works
+
+```
+┌─────────────┐                    ┌─────────────┐
+│  Server A   │                    │  Server B   │
+│  client-1   │                    │  client-2   │
+└──────┬──────┘                    └──────▲──────┘
+       │                                  │
+       │ 1. sendTo("client-2")            │
+       │    → lookup: client-2 → srv-B    │
+       │                                  │
+       │ 2. channels.push("srv-B", msg)   │
+       └──────────┬───────────────────────┘
+                  │
+           ┌──────▼──────┐
+           │   Database  │
+           │ (message    │
+           │   bus)      │
+           └─────────────┘
+```
+
+APE creates its own namespaced keys/tables (`ape:*` or `ape_*`). No schema conflicts with your data.
+
+👉 **[See detailed Forest documentation](server/README.md#forest-distributed-mesh)**
 
 ---
 
@@ -131,7 +182,7 @@ Initialize api-ape on a Node.js HTTP/HTTPS server.
 |--------|------|-------------|
 | `server` | `http.Server` | Node.js HTTP or HTTPS server instance |
 | `where` | `string` | Directory containing controller files (default: `'api'`) |
-| `onConnent` | `function` | Connection lifecycle hook (see [Connection Lifecycle](#connection-lifecycle)) |
+| `onConnect` | `function` | Connection lifecycle hook (see [Connection Lifecycle](#connection-lifecycle)) |
 
 #### Controller Context (`this`)
 
@@ -141,14 +192,24 @@ Inside controller functions, `this` provides:
 |----------|-------------|
 | `this.broadcast(type, data)` | Send to **ALL** connected clients |
 | `this.broadcastOthers(type, data)` | Send to all **EXCEPT** the caller |
-| `this.online()` | Get count of connected clients |
-| `this.getClients()` | Get array of connected clientIds |
 | `this.clientId` | Unique ID of the calling client (generated by api-ape) |
 | `this.sessionId` | Session ID from cookie (set by outer framework, may be `null`) |
 | `this.req` | Original HTTP request |
 | `this.socket` | WebSocket instance |
 | `this.agent` | Parsed user-agent (browser, OS, device) |
 
+#### `ape.clients`
+
+A read-only Map of connected clients. Each client provides:
+
+| Property | Description |
+|----------|-------------|
+| `clientId` | Unique client identifier |
+| `sessionId` | Session ID from cookie (may be `null`) |
+| `embed` | Embedded values from onConnect |
+| `agent` | Parsed user-agent (browser, OS, device) |
+| `sendTo(type, data)` | Send a message to this specific client |
+
 ### Client
 
 #### `api.<path>.<method>(...args)`
@@ -178,13 +239,12 @@ api.on('notification', ({ data, err, type }) => {
 })
 ```
 
-#### `api.getTransport()`
+#### `api.transport`
 
-Get the currently active transport type.
+Get the currently active transport type. This is a read-only property.
 
 ```js
-const transport = api.getTransport()
-console.log(transport) // 'websocket' | 'polling' | null
+console.log(api.transport) // 'websocket' | 'polling' | null
 ```
 
 #### `api.onConnectionChange(handler)`
@@ -229,7 +289,7 @@ Customize behavior per connection:
 ```js
 ape(server, {
   where: 'api',
-  onConnent(socket, req, clientId) {
+  onConnect(socket, req, clientId) {
     return {
       // Embed values into `this` for all controllers
       embed: {
@@ -249,7 +309,7 @@ ape(server, {
       },
       
       onError: (errStr) => console.error(errStr),
-      onDisconnent: () => console.log('Client left')
+      onDisconnect: () => console.log('Client left')
     }
   }
 })
@@ -281,16 +341,27 @@ module.exports = function(announcement) {
 }
 ```
 
-### Get Online Count
+### Using ape.clients
 
 ```js
-// api/stats.js
-module.exports = function() {
-  return {
-    online: this.online(),
-    clients: this.getClients()
-  }
+const ape = require('api-ape')
+
+// Get online count
+console.log('Online:', ape.clients.size)
+
+// Get all client IDs
+const clientIds = Array.from(ape.clients.keys())
+
+// Send to a specific client
+const client = ape.clients.get(someClientId)
+if (client) {
+  client.sendTo('notification', { message: 'Hello!' })
 }
+
+// Iterate all clients
+ape.clients.forEach((client, clientId) => {
+  console.log(`Client ${clientId}:`, client.sessionId, client.agent.browser.name)
+})
 ```
 
 ### Access Request Data
@@ -454,7 +525,7 @@ This prevents malicious sites from making requests to your api-ape server while
 ### Security Considerations
 
 * Validate all input in controllers
-* Use authentication/authorization in `onConnent` hooks
+* Use authentication/authorization in `onConnect` hooks
 * Sanitize data before broadcasting
 * Keep dependencies up to date
 

package/client/browser.js

@@ -1,25 +1,25 @@
 import connectSocket from './connectSocket.js'
 
-const { sender, setOnReciver, onConnectionChange, getTransport } = connectSocket()
+const client = connectSocket()
 connectSocket.autoReconnect()
 
 // Global API - use defineProperty to bypass Proxy interception
-window.api = sender
+window.api = client.sender
 Object.defineProperty(window.api, 'on', {
-    value: setOnReciver,
+    value: client.setOnReceiver,
     writable: false,
     enumerable: false,
     configurable: false
 })
 Object.defineProperty(window.api, 'onConnectionChange', {
-    value: onConnectionChange,
+    value: client.onConnectionChange,
     writable: false,
     enumerable: false,
     configurable: false
 })
-Object.defineProperty(window.api, 'getTransport', {
-    value: getTransport,
-    writable: false,
+// Read-only transport property - only ape can change this internally
+Object.defineProperty(window.api, 'transport', {
+    get: () => client.transport,
     enumerable: false,
     configurable: false
 })

package/client/connectSocket.js

@@ -160,13 +160,13 @@ function getSocketUrl() {
 }
 
 let reconnect = false
-const connentTimeout = 5000
+const connectTimeout = 5000
 const totalRequestTimeout = 10000
 //const location = window.location
 
 const joinKey = "/"
 // Properties accessed directly on `ape` that should NOT be intercepted
-const reservedKeys = new Set(['on', 'onConnectionChange', 'getTransport'])
+const reservedKeys = new Set(['on', 'onConnectionChange', 'transport'])
 const handler = {
   get(fn, key) {
     // Skip proxy interception for reserved keys - return actual property
@@ -193,10 +193,9 @@ function wrap(api) {
 
 let __socket = false, ready = false, wsSend = false;
 const waitingOn = {};
-const reciverOn = [];
 
 let aWaitingSend = []
-const reciverOnAr = [];
+const receiverArray = [];
 const ofTypesOb = {};
 
 /**
@@ -213,12 +212,23 @@ function switchToStreaming() {
     streamingTransport.onMessage = async (msg) => {
       const { err, type, data } = msg
 
+      // Process linked resources and shared files
+      let processedData = data
+      if (data && !err) {
+        try {
+          processedData = await fetchLinkedResources(data)
+          processedData = await fetchSharedFiles(processedData)
+        } catch (fetchErr) {
+          console.error(`🦍 Failed to hydrate streaming data:`, fetchErr)
+        }
+      }
+
       // Dispatch to type-specific handlers
       if (ofTypesOb[type]) {
-        ofTypesOb[type].forEach(worker => worker({ err, type, data }))
+        ofTypesOb[type].forEach(worker => worker({ err, type, data: processedData }))
       }
       // Dispatch to general handlers
-      reciverOnAr.forEach(worker => worker({ err, type, data }))
+      receiverArray.forEach(worker => worker({ err, type, data: processedData }))
     }
 
     streamingTransport.onOpen = () => {
@@ -227,11 +237,11 @@ function switchToStreaming() {
       console.log('🦍 HTTP streaming connected')
 
       // Flush waiting messages
-      aWaitingSend.forEach(({ type, data, next, err, waiting, createdAt, timer }) => {
+      aWaitingSend.forEach(({ type, data, resolve, reject, waiting, createdAt, timer }) => {
         clearTimeout(timer)
         const resultPromise = streamingSend(type, data, createdAt)
         if (waiting) {
-          resultPromise.then(next).catch(err)
+          resultPromise.then(resolve).catch(reject)
         }
       })
       aWaitingSend = []
@@ -319,11 +329,11 @@ function tryWebSocket(isRetry = false) {
     ready = true
     notifyConnectionChange(ConnectionState.Connected)
 
-    aWaitingSend.forEach(({ type, data, next, err, waiting, createdAt, timer }) => {
+    aWaitingSend.forEach(({ type, data, resolve, reject, waiting, createdAt, timer }) => {
       clearTimeout(timer)
       const resultPromise = wsSend(type, data, createdAt)
       if (waiting) {
-        resultPromise.then(next).catch(err)
+        resultPromise.then(resolve).catch(reject)
       }
     })
     aWaitingSend = []
@@ -338,7 +348,8 @@ function tryWebSocket(isRetry = false) {
         // Check for linked resources and fetch them before resolving
         if (data && !err) {
           try {
-            const hydratedData = await fetchLinkedResources(data)
+            let hydratedData = await fetchLinkedResources(data)
+            hydratedData = await fetchSharedFiles(hydratedData)
             waitingOn[queryId](err, hydratedData)
           } catch (fetchErr) {
             waitingOn[queryId](fetchErr, null)
@@ -353,11 +364,12 @@ function tryWebSocket(isRetry = false) {
       return
     }
 
-    // Only messages WITHOUT queryId go to setOnReciver
+    // Only messages WITHOUT queryId go to setOnReceiver
     let processedData = data
     if (data && !err) {
       try {
         processedData = await fetchLinkedResources(data)
+        processedData = await fetchSharedFiles(processedData)
       } catch (fetchErr) {
         console.error(`🦍 Failed to hydrate broadcast data:`, fetchErr)
       }
@@ -366,7 +378,7 @@ function tryWebSocket(isRetry = false) {
     if (ofTypesOb[type]) {
       ofTypesOb[type].forEach(worker => worker({ err, type, data: processedData }))
     }
-    reciverOnAr.forEach(worker => worker({ err, type, data: processedData }))
+    receiverArray.forEach(worker => worker({ err, type, data: processedData }))
   }
 
   ws.onerror = function (err) {
@@ -429,6 +441,134 @@ function findLinkedResources(obj, path = '') {
   return resources
 }
 
+/**
+ * Find all F-tagged (shared file) properties in data
+ * Returns array of { path, hash, originalKey }
+ */
+function findFileTags(obj, path = '') {
+  const files = []
+
+  if (obj === null || obj === undefined || typeof obj !== 'object') {
+    return files
+  }
+
+  if (Array.isArray(obj)) {
+    for (let i = 0; i < obj.length; i++) {
+      files.push(...findFileTags(obj[i], path ? `${path}.${i}` : String(i)))
+    }
+    return files
+  }
+
+  for (const key of Object.keys(obj)) {
+    // Check for F-tag in key (client-to-client shared file marker)
+    if (key.endsWith('<!F>')) {
+      const cleanKey = key.slice(0, -4)
+      const hash = obj[key]
+      files.push({
+        path: path ? `${path}.${cleanKey}` : cleanKey,
+        hash,
+        originalKey: key
+      })
+    } else {
+      files.push(...findFileTags(obj[key], path ? `${path}.${key}` : key))
+    }
+  }
+
+  return files
+}
+
+/**
+ * Clean up F-tagged keys (rename key<!F> to key)
+ */
+function cleanFileTags(obj) {
+  if (obj === null || obj === undefined || typeof obj !== 'object') {
+    return obj
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(cleanFileTags)
+  }
+
+  const cleaned = {}
+  for (const key of Object.keys(obj)) {
+    if (key.endsWith('<!F>')) {
+      const cleanKey = key.slice(0, -4)
+      cleaned[cleanKey] = obj[key]
+    } else {
+      cleaned[key] = cleanFileTags(obj[key])
+    }
+  }
+  return cleaned
+}
+
+/**
+ * Fetch shared files (client-to-client transfers)
+ * Retries if upload is still in progress
+ */
+async function fetchSharedFiles(data, maxRetries = 5) {
+  const files = findFileTags(data)
+
+  if (files.length === 0) {
+    return data
+  }
+
+  console.log(`🦍 Fetching ${files.length} shared file(s)`)
+
+  const cleanedData = cleanFileTags(data)
+
+  const hostname = window.location.hostname
+  const isLocal = ["localhost", "127.0.0.1", "[::1]"].includes(hostname)
+  const isHttps = window.location.protocol === "https:"
+  const port = isLocal ? 9010 : (window.location.port || (isHttps ? 443 : 80))
+  const protocol = isHttps ? "https" : "http"
+  const portSuffix = (isLocal || (port !== 80 && port !== 443)) ? `:${port}` : ""
+  const baseUrl = `${protocol}://${hostname}${portSuffix}`
+
+  await Promise.all(files.map(async ({ path, hash }) => {
+    let retries = 0
+    let backoff = 100 // Start with 100ms
+
+    while (retries < maxRetries) {
+      try {
+        const response = await fetch(`${baseUrl}/api/ape/data/${hash}`, {
+          credentials: 'include'
+        })
+
+        if (!response.ok) {
+          // 404 might mean file not uploaded yet, retry
+          if (response.status === 404 && retries < maxRetries - 1) {
+            retries++
+            await new Promise(r => setTimeout(r, backoff))
+            backoff *= 2 // Exponential backoff
+            continue
+          }
+          throw new Error(`Failed to fetch shared file: ${response.status}`)
+        }
+
+        const arrayBuffer = await response.arrayBuffer()
+        setValueAtPath(cleanedData, path, arrayBuffer)
+
+        // Check if upload is still in progress
+        const isComplete = response.headers.get('X-Ape-Complete') === '1'
+        if (!isComplete) {
+          console.log(`🦍 Shared file ${hash} still uploading (${response.headers.get('X-Ape-Total-Received') || '?'} bytes)`)
+        }
+        break
+      } catch (err) {
+        if (retries >= maxRetries - 1) {
+          console.error(`🦍 Failed to fetch shared file at ${path}:`, err)
+          setValueAtPath(cleanedData, path, null)
+        }
+        retries++
+        await new Promise(r => setTimeout(r, backoff))
+        backoff *= 2
+      }
+    }
+  }))
+
+  return cleanedData
+}
+
 /**
  * Set a value at a nested path in an object
  */
@@ -667,6 +807,101 @@ function processBinaryForUpload(data, path = '') {
   return { processedData: data, uploads: [] }
 }
 
+/**
+ * Find and extract binary data for SHARING (client-to-client)
+ * Uses <!F> tag instead of <!A>/<!B>
+ * Returns { processedData, shares: [{ path, hash, data }] }
+ */
+function processBinaryForSharing(data, path = '') {
+  if (data === null || data === undefined) {
+    return { processedData: data, shares: [] }
+  }
+
+  if (isBinaryData(data)) {
+    const hash = generateUploadHash(path || 'share')
+    return {
+      processedData: { [`__ape_share__`]: hash },
+      shares: [{ path, hash, data }]
+    }
+  }
+
+  if (Array.isArray(data)) {
+    const processedArray = []
+    const allShares = []
+
+    for (let i = 0; i < data.length; i++) {
+      const itemPath = path ? `${path}.${i}` : String(i)
+      const { processedData, shares } = processBinaryForSharing(data[i], itemPath)
+      processedArray.push(processedData)
+      allShares.push(...shares)
+    }
+
+    return { processedData: processedArray, shares: allShares }
+  }
+
+  if (typeof data === 'object') {
+    const processedObj = {}
+    const allShares = []
+
+    for (const key of Object.keys(data)) {
+      const itemPath = path ? `${path}.${key}` : key
+      const { processedData, shares } = processBinaryForSharing(data[key], itemPath)
+
+      // If this was binary data, mark the key with <!F> tag
+      if (shares.length > 0 && processedData?.__ape_share__) {
+        processedObj[`${key}<!F>`] = processedData.__ape_share__
+      } else {
+        processedObj[key] = processedData
+      }
+      allShares.push(...shares)
+    }
+
+    return { processedData: processedObj, shares: allShares }
+  }
+
+  return { processedData: data, shares: [] }
+}
+
+/**
+ * Upload shared files via HTTP PUT
+ * Uses different endpoint pattern for streaming files
+ */
+async function uploadSharedFiles(shares) {
+  if (shares.length === 0) return
+
+  // Build base URL
+  const hostname = window.location.hostname
+  const isLocal = ["localhost", "127.0.0.1", "[::1]"].includes(hostname)
+  const isHttps = window.location.protocol === "https:"
+  const port = isLocal ? 9010 : (window.location.port || (isHttps ? 443 : 80))
+  const protocol = isHttps ? "https" : "http"
+  const portSuffix = (isLocal || (port !== 80 && port !== 443)) ? `:${port}` : ""
+  const baseUrl = `${protocol}://${hostname}${portSuffix}`
+
+  console.log(`🦍 Uploading ${shares.length} shared file(s)`)
+
+  await Promise.all(shares.map(async ({ hash, data }) => {
+    try {
+      // For shared files, use upload pattern with hash as both queryId and pathHash
+      const response = await fetch(`${baseUrl}/api/ape/data/_share/${hash}`, {
+        method: 'PUT',
+        credentials: 'include',
+        headers: {
+          'Content-Type': 'application/octet-stream'
+        },
+        body: data
+      })
+
+      if (!response.ok) {
+        throw new Error(`Shared upload failed: ${response.status}`)
+      }
+    } catch (err) {
+      console.error(`🦍 Failed to upload shared file ${hash}:`, err)
+      throw err
+    }
+  }))
+}
+
 /**
  * Upload binary data via HTTP PUT
  */
@@ -779,19 +1014,19 @@ const sender = (type, data) => {
     return wsSend(type, data, createdAt, true)
   }
 
-  const timeLetForReqToBeMade = (createdAt + connentTimeout) - Date.now() // 5sec for reconnent
+  const timeLetForReqToBeMade = (createdAt + connectTimeout) - Date.now() // 5sec for reconnect
 
   const timer = setTimeout(() => {
     const errMessage = "Request not sent for :" + type
     if (payload.waiting) {
-      payload.err(new Error(errMessage))
+      payload.reject(new Error(errMessage))
     } else {
       throw new Error(errMessage)
     }
   }, timeLetForReqToBeMade);
 
-  const payload = { type, data, next: undefined, err: undefined, waiting: false, createdAt, timer };
-  const waitingOnOpen = new Promise((res, er) => { payload.next = res; payload.err = er; })
+  const payload = { type, data, resolve: undefined, reject: undefined, waiting: false, createdAt, timer };
+  const waitingOnOpen = new Promise((res, rej) => { payload.resolve = res; payload.reject = rej; })
 
   const waitingOnOpenThen = waitingOnOpen.then;
   const waitingOnOpenCatch = waitingOnOpen.catch;
@@ -822,14 +1057,14 @@ const sender = (type, data) => {
 function buildClientInterface() {
   return {
     sender: wrap(sender),
-    setOnReciver: (onTypeStFn, handlerFn) => {
+    setOnReceiver: (onTypeStFn, handlerFn) => {
       if ("string" === typeof onTypeStFn) {
         // Replace handler for this type (prevents duplicates in React StrictMode)
         ofTypesOb[onTypeStFn] = [handlerFn]
       } else {
         // For general receivers, prevent duplicates by checking
-        if (!reciverOnAr.includes(onTypeStFn)) {
-          reciverOnAr.push(onTypeStFn)
+        if (!receiverArray.includes(onTypeStFn)) {
+          receiverArray.push(onTypeStFn)
         }
       }
     },
@@ -843,8 +1078,8 @@ function buildClientInterface() {
         if (idx > -1) connectionChangeListeners.splice(idx, 1)
       }
     },
-    // Expose current transport type
-    getTransport: () => currentTransport
+    // Expose current transport type (read-only)
+    get transport() { return currentTransport }
   }
 }
 

package/client/index.js

@@ -55,7 +55,7 @@ function getClient() {
 
         // Flush buffered receivers
         bufferedReceivers.forEach(({ type, handler }) => {
-            client.setOnReciver(type, handler)
+            client.setOnReceiver(type, handler)
         })
         bufferedReceivers.length = 0
 
@@ -88,7 +88,7 @@ const senderProxy = new Proxy({}, {
         // Reserved properties
         if (prop === 'on') return on
         if (prop === 'onConnectionChange') return onConnectionChange
-        if (prop === 'getTransport') return () => resolvedClient?.getTransport?.() || null
+        if (prop === 'transport') return resolvedClient?.transport || null
         if (prop === 'then' || prop === 'catch') return undefined // Not a Promise
 
         // Return a function that either calls directly or buffers
@@ -115,7 +115,7 @@ const senderProxy = new Proxy({}, {
  */
 function on(type, handler) {
     if (resolvedClient) {
-        resolvedClient.setOnReciver(type, handler)
+        resolvedClient.setOnReceiver(type, handler)
     } else {
         bufferedReceivers.push({ type, handler })
         getClient()