api-ape 1.0.1 → 1.1.0

package/server/lib/fileTransfer.js

@@ -0,0 +1,247 @@
+/**
+ * FileTransferManager - Handles temporary binary data endpoints
+ * 
+ * For downloads (server → client):
+ * - Registers binary data with a hash
+ * - Creates temporary endpoint at GET /api/ape/data/:hash
+ * - Verifies session before allowing download
+ * - Auto-cleanup after timeout
+ * 
+ * For uploads (client → server):
+ * - Registers upload expectation with queryId + pathHash
+ * - Receives data via PUT /api/ape/data/:queryId/:pathHash
+ * - Waits for matching WS message before processing
+ */
+
+// Default timeouts (configurable)
+const DEFAULT_START_TIMEOUT = 60 * 1000  // 1 minute to start download
+const DEFAULT_COMPLETE_TIMEOUT = 60 * 1000  // 1 minute after download starts
+
+class FileTransferManager {
+    constructor(options = {}) {
+        this.startTimeout = options.startTimeout || DEFAULT_START_TIMEOUT
+        this.completeTimeout = options.completeTimeout || DEFAULT_COMPLETE_TIMEOUT
+
+        // Map<hash, { data, contentType, sessionHostId, createdAt, downloadStarted, timer }>
+        this.pendingDownloads = new Map()
+
+        // Map<`${queryId}/${pathHash}`, { sessionHostId, createdAt, resolver, rejector, timer, data }>
+        this.pendingUploads = new Map()
+
+        // Cleanup interval
+        this._cleanupInterval = setInterval(() => this._cleanup(), 30000)
+    }
+
+    /**
+     * Register a binary download
+     * @param {string} hash - Unique hash for this download
+     * @param {Buffer|ArrayBuffer} data - Binary data to serve
+     * @param {string} contentType - MIME type (e.g., 'application/octet-stream')
+     * @param {string} sessionHostId - Host ID of the client session
+     * @returns {string} The hash (for confirmation)
+     */
+    registerDownload(hash, data, contentType, sessionHostId) {
+        // Clear any existing entry with same hash
+        if (this.pendingDownloads.has(hash)) {
+            const existing = this.pendingDownloads.get(hash)
+            if (existing.timer) clearTimeout(existing.timer)
+        }
+
+        const entry = {
+            data,
+            contentType: contentType || 'application/octet-stream',
+            sessionHostId,
+            createdAt: Date.now(),
+            downloadStarted: false,
+            timer: setTimeout(() => {
+                // Auto-remove if download never started
+                if (!this.pendingDownloads.get(hash)?.downloadStarted) {
+                    this.pendingDownloads.delete(hash)
+                    console.log(`📦 Download expired (never started): ${hash}`)
+                }
+            }, this.startTimeout)
+        }
+
+        this.pendingDownloads.set(hash, entry)
+        console.log(`📦 Registered download: ${hash} for session ${sessionHostId}`)
+        return hash
+    }
+
+    /**
+     * Get download data (called by HTTP handler)
+     * @param {string} hash - Download hash
+     * @param {string} requestingHostId - Host ID of requester (from session/cookie)
+     * @returns {{ data: Buffer, contentType: string } | null}
+     */
+    getDownload(hash, requestingHostId) {
+        const entry = this.pendingDownloads.get(hash)
+
+        if (!entry) {
+            console.warn(`📦 Download not found: ${hash}`)
+            return null
+        }
+
+        // Session verification
+        if (entry.sessionHostId !== requestingHostId) {
+            console.warn(`📦 Session mismatch for ${hash}: expected ${entry.sessionHostId}, got ${requestingHostId}`)
+            return null
+        }
+
+        // Mark download as started
+        if (!entry.downloadStarted) {
+            entry.downloadStarted = true
+            clearTimeout(entry.timer)
+
+            // Set new timer for cleanup after completion
+            entry.timer = setTimeout(() => {
+                this.pendingDownloads.delete(hash)
+                console.log(`📦 Download cleaned up: ${hash}`)
+            }, this.completeTimeout)
+        }
+
+        return {
+            data: entry.data,
+            contentType: entry.contentType
+        }
+    }
+
+    /**
+     * Register an expected upload
+     * @param {string} queryId - Query ID from WS message
+     * @param {string} pathHash - Hash of property path
+     * @param {string} sessionHostId - Host ID of the client session
+     * @returns {Promise<Buffer>} Resolves when upload is received
+     */
+    registerUpload(queryId, pathHash, sessionHostId) {
+        const key = `${queryId}/${pathHash}`
+
+        return new Promise((resolve, reject) => {
+            const entry = {
+                sessionHostId,
+                createdAt: Date.now(),
+                resolver: resolve,
+                rejector: reject,
+                data: null,
+                timer: setTimeout(() => {
+                    this.pendingUploads.delete(key)
+                    reject(new Error(`Upload timeout: ${key}`))
+                }, this.startTimeout)
+            }
+
+            this.pendingUploads.set(key, entry)
+            console.log(`📤 Registered upload expectation: ${key} for session ${sessionHostId}`)
+        })
+    }
+
+    /**
+     * Receive upload data (called by HTTP handler)
+     * @param {string} queryId - Query ID from URL
+     * @param {string} pathHash - Path hash from URL
+     * @param {Buffer} data - Uploaded binary data
+     * @param {string} requestingHostId - Host ID of uploader
+     * @returns {boolean} True if accepted
+     */
+    receiveUpload(queryId, pathHash, data, requestingHostId) {
+        const key = `${queryId}/${pathHash}`
+        const entry = this.pendingUploads.get(key)
+
+        if (!entry) {
+            console.warn(`📤 Upload not expected: ${key}`)
+            return false
+        }
+
+        // Session verification
+        if (entry.sessionHostId !== requestingHostId) {
+            console.warn(`📤 Session mismatch for upload ${key}: expected ${entry.sessionHostId}, got ${requestingHostId}`)
+            return false
+        }
+
+        // Clear timeout and resolve
+        clearTimeout(entry.timer)
+        entry.resolver(data)
+        this.pendingUploads.delete(key)
+        console.log(`📤 Upload received: ${key}`)
+
+        return true
+    }
+
+    /**
+     * Generate hash for download from queryId and property path
+     * @param {string} queryId - The query ID
+     * @param {string} propertyPath - The property path (e.g., 'files.0.data')
+     * @returns {string} Combined hash
+     */
+    static generateHash(queryId, propertyPath) {
+        // Simple hash combining queryId and path
+        // In production, could use crypto.createHash
+        const combined = `${queryId}:${propertyPath}`
+        let hash = 0
+        for (let i = 0; i < combined.length; i++) {
+            const char = combined.charCodeAt(i)
+            hash = ((hash << 5) - hash) + char
+            hash = hash & hash // Convert to 32bit integer
+        }
+        return Math.abs(hash).toString(36)
+    }
+
+    /**
+     * Cleanup expired entries
+     * @private
+     */
+    _cleanup() {
+        const now = Date.now()
+        const maxAge = this.startTimeout + this.completeTimeout
+
+        // Cleanup downloads
+        for (const [hash, entry] of this.pendingDownloads) {
+            if (now - entry.createdAt > maxAge) {
+                clearTimeout(entry.timer)
+                this.pendingDownloads.delete(hash)
+                console.log(`📦 Cleanup stale download: ${hash}`)
+            }
+        }
+
+        // Cleanup uploads
+        for (const [key, entry] of this.pendingUploads) {
+            if (now - entry.createdAt > maxAge) {
+                clearTimeout(entry.timer)
+                entry.rejector(new Error(`Upload expired: ${key}`))
+                this.pendingUploads.delete(key)
+                console.log(`📤 Cleanup stale upload: ${key}`)
+            }
+        }
+    }
+
+    /**
+     * Shutdown cleanup
+     */
+    destroy() {
+        clearInterval(this._cleanupInterval)
+
+        // Clear all timers
+        for (const entry of this.pendingDownloads.values()) {
+            clearTimeout(entry.timer)
+        }
+        for (const entry of this.pendingUploads.values()) {
+            clearTimeout(entry.timer)
+        }
+
+        this.pendingDownloads.clear()
+        this.pendingUploads.clear()
+    }
+}
+
+// Singleton instance
+let instance = null
+
+function getFileTransferManager(options) {
+    if (!instance) {
+        instance = new FileTransferManager(options)
+    }
+    return instance
+}
+
+module.exports = {
+    FileTransferManager,
+    getFileTransferManager
+}

package/server/lib/main.js

@@ -2,10 +2,11 @@ const loader = require('./loader')
 const wiring = require('./wiring')
 const expressWs = require('express-ws');
 const path = require('path');
+const { getFileTransferManager } = require('./fileTransfer');
 
 let created = false
 
-module.exports = function (app, { where, onConnent }) {
+module.exports = function (app, { where, onConnent, fileTransferOptions }) {
 
     if (created) {
         throw new Error("Api-Ape already started")
@@ -14,10 +15,77 @@ module.exports = function (app, { where, onConnent }) {
     expressWs(app)
     const controllers = loader(where)
 
+    // Initialize file transfer manager
+    const fileTransfer = getFileTransferManager(fileTransferOptions)
+
     // Serve bundled client at /ape.js
     app.get('/api/ape.js', (req, res) => {
         res.sendFile(path.join(__dirname, '../../dist/ape.js'))
     })
 
-    app.ws('/api/ape', wiring(controllers, onConnent))
+    // File download endpoint - GET /api/ape/data/:hash
+    app.get('/api/ape/data/:hash', (req, res) => {
+        const { hash } = req.params
+
+        // Get hostId from session/cookie (set during WS connection)
+        const hostId = req.cookies?.apeHostId || req.headers['x-ape-host-id']
+
+        if (!hostId) {
+            return res.status(401).json({ error: 'Missing session identifier' })
+        }
+
+        // Enforce HTTPS in production
+        const isLocal = ['localhost', '127.0.0.1', '[::1]'].includes(req.hostname)
+        if (!isLocal && !req.secure && req.get('x-forwarded-proto') !== 'https') {
+            return res.status(403).json({ error: 'HTTPS required for file transfers' })
+        }
+
+        const result = fileTransfer.getDownload(hash, hostId)
+
+        if (!result) {
+            return res.status(404).json({ error: 'Download not found or unauthorized' })
+        }
+
+        res.set('Content-Type', result.contentType)
+        res.set('Content-Length', result.data.length || result.data.byteLength)
+        res.send(result.data)
+    })
+
+    // File upload endpoint - PUT /api/ape/data/:queryId/:pathHash
+    app.put('/api/ape/data/:queryId/:pathHash', (req, res) => {
+        const { queryId, pathHash } = req.params
+
+        // Get hostId from session/cookie
+        const hostId = req.cookies?.apeHostId || req.headers['x-ape-host-id']
+
+        if (!hostId) {
+            return res.status(401).json({ error: 'Missing session identifier' })
+        }
+
+        // Enforce HTTPS in production
+        const isLocal = ['localhost', '127.0.0.1', '[::1]'].includes(req.hostname)
+        if (!isLocal && !req.secure && req.get('x-forwarded-proto') !== 'https') {
+            return res.status(403).json({ error: 'HTTPS required for file transfers' })
+        }
+
+        // Collect body as buffer
+        const chunks = []
+        req.on('data', chunk => chunks.push(chunk))
+        req.on('end', () => {
+            const data = Buffer.concat(chunks)
+            const success = fileTransfer.receiveUpload(queryId, pathHash, data, hostId)
+
+            if (success) {
+                res.json({ success: true })
+            } else {
+                res.status(404).json({ error: 'Upload not expected or unauthorized' })
+            }
+        })
+        req.on('error', (err) => {
+            res.status(500).json({ error: err.message })
+        })
+    })
+
+    // Pass fileTransfer to wiring so send.js can register downloads
+    app.ws('/api/ape', wiring(controllers, onConnent, fileTransfer))
 }

package/server/lib/wiring.js

@@ -26,7 +26,7 @@ function defaultEvents(events = {}) {
 //============================================== wiring
 //=====================================================
 
-module.exports = function wiring(controllers, onConnent) {
+module.exports = function wiring(controllers, onConnent, fileTransfer) {
     onConnent = onConnent || (() => { });
     return function webSocketHandler(socket, req) {
 
@@ -70,7 +70,8 @@ module.exports = function wiring(controllers, onConnent) {
                     events: { onReceive, onSend, onError, onDisconnent },
                     controllers,
                     sharedValues,
-                    embedValues: embed
+                    embedValues: embed,
+                    fileTransfer  // Pass file transfer manager
                 }// END ape
                 send = socketSend(ape)
                 ape.send = send
@@ -92,3 +93,4 @@ module.exports = function wiring(controllers, onConnent) {
 
     } // END webSocketHandler
 } // END wiring
+

package/server/socket/receive.js

@@ -2,8 +2,95 @@ const messageHash = require('../../utils/messageHash')
 const { broadcast, online, getClients } = require('../lib/broadcast')
 const jss = require('../../utils/jss')
 
+/**
+ * Find B/A tagged properties in data (indicating pending uploads)
+ * Returns array of { path, hash, tag }
+ */
+function findUploadTags(obj, path = '') {
+    const uploads = []
+
+    if (obj === null || obj === undefined || typeof obj !== 'object') {
+        return uploads
+    }
+
+    if (Array.isArray(obj)) {
+        for (let i = 0; i < obj.length; i++) {
+            uploads.push(...findUploadTags(obj[i], path ? `${path}.${i}` : String(i)))
+        }
+        return uploads
+    }
+
+    for (const key of Object.keys(obj)) {
+        // Check for B or A tag (binary upload markers)
+        const bMatch = key.match(/^(.+)<!B>$/)
+        const aMatch = key.match(/^(.+)<!A>$/)
+
+        if (bMatch) {
+            uploads.push({
+                path: path ? `${path}.${bMatch[1]}` : bMatch[1],
+                hash: obj[key],
+                tag: 'B',
+                originalKey: key
+            })
+        } else if (aMatch) {
+            uploads.push({
+                path: path ? `${path}.${aMatch[1]}` : aMatch[1],
+                hash: obj[key],
+                tag: 'A',
+                originalKey: key
+            })
+        } else {
+            uploads.push(...findUploadTags(obj[key], path ? `${path}.${key}` : key))
+        }
+    }
+
+    return uploads
+}
+
+/**
+ * Clean upload tags from data (rename key<!B> to key)
+ */
+function cleanUploadTags(obj) {
+    if (obj === null || obj === undefined || typeof obj !== 'object') {
+        return obj
+    }
+
+    if (Array.isArray(obj)) {
+        return obj.map(cleanUploadTags)
+    }
+
+    const cleaned = {}
+    for (const key of Object.keys(obj)) {
+        const bMatch = key.match(/^(.+)<!B>$/)
+        const aMatch = key.match(/^(.+)<!A>$/)
+
+        if (bMatch) {
+            cleaned[bMatch[1]] = obj[key] // Will be replaced with actual data
+        } else if (aMatch) {
+            cleaned[aMatch[1]] = obj[key] // Will be replaced with actual data
+        } else {
+            cleaned[key] = cleanUploadTags(obj[key])
+        }
+    }
+    return cleaned
+}
+
+/**
+ * Set value at nested path
+ */
+function setValueAtPath(obj, path, value) {
+    const parts = path.split('.')
+    let current = obj
+
+    for (let i = 0; i < parts.length - 1; i++) {
+        current = current[parts[i]]
+    }
+
+    current[parts[parts.length - 1]] = value
+}
+
 module.exports = function receiveHandler(ape) {
-    const { send, checkReply, events, controllers, sharedValues, hostId, embedValues } = ape
+    const { send, checkReply, events, controllers, sharedValues, hostId, embedValues, fileTransfer } = ape
 
     // Build `this` context for controllers
     // Includes: client metadata + api-ape utilities
@@ -18,7 +105,7 @@ module.exports = function receiveHandler(ape) {
         hostId
     }
 
-    return function onReceive(msg) {
+    return async function onReceive(msg) {
         // Convert Buffer to string - WebSocket messages may arrive as binary
         const msgString = typeof msg === 'string' ? msg : msg.toString('utf8');
         const queryId = messageHash(msgString);
@@ -31,6 +118,34 @@ module.exports = function receiveHandler(ape) {
             // Call onReceive hook - it should return a finish callback
             const onFinish = events.onReceive(queryId, data, type) || (() => { })
 
+            // Check for pending uploads (B/A tags)
+            let processedData = data
+            if (fileTransfer && data) {
+                const uploadTags = findUploadTags(data)
+
+                if (uploadTags.length > 0) {
+                    console.log(`📤 Waiting for ${uploadTags.length} upload(s) for ${type}`)
+
+                    // Clean the data object
+                    processedData = cleanUploadTags(data)
+
+                    // Wait for all uploads
+                    try {
+                        await Promise.all(uploadTags.map(async ({ path, hash }) => {
+                            const uploadData = await fileTransfer.registerUpload(queryId, hash, hostId)
+                            setValueAtPath(processedData, path, uploadData)
+                        }))
+                    } catch (uploadErr) {
+                        console.error(`📤 Upload wait failed:`, uploadErr)
+                        send(queryId, false, false, uploadErr)
+                        if (typeof onFinish === 'function') {
+                            onFinish(uploadErr, true)
+                        }
+                        return
+                    }
+                }
+            }
+
             const result = new Promise((resolve, reject) => {
                 try {
                     const controller = controllers[type]
@@ -38,7 +153,7 @@ module.exports = function receiveHandler(ape) {
                         throw `TypeError: "${type}" was not found`
                     }
                     checkReply(queryId, createdAt)
-                    resolve(controller.call(that, data))
+                    resolve(controller.call(that, processedData))
                 } catch (err) {
                     reject(err)
                 }

package/server/socket/send.js

@@ -1,4 +1,5 @@
 const jss = require('../../utils/jss')
+const { FileTransferManager } = require('../lib/fileTransfer')
 
 function checkSocketState(socket) {
   if (socket.readyState !== socket.OPEN) {
@@ -17,7 +18,88 @@ function checkSocketState(socket) {
   } // END if
 } // END checkSocketState
 
-module.exports = function sendHandler({ socket, events, hostId }) {
+/**
+ * Check if value is binary data (Buffer, ArrayBuffer, or typed array)
+ */
+function isBinaryData(value) {
+  if (value === null || value === undefined) return false
+  return Buffer.isBuffer(value) ||
+    value instanceof ArrayBuffer ||
+    ArrayBuffer.isView(value)
+}
+
+/**
+ * Detect content type from binary data
+ */
+function detectContentType(data) {
+  // Could be enhanced with magic number detection
+  return 'application/octet-stream'
+}
+
+/**
+ * Process data object, replacing binary values with L-tagged hashes
+ * Returns { processedData, binaryEntries }
+ */
+function processBinaryData(data, queryId, fileTransfer, hostId, path = '') {
+  if (data === null || data === undefined) {
+    return { processedData: data, binaryEntries: [] }
+  }
+
+  if (isBinaryData(data)) {
+    // This is binary data - register and return hash
+    const hash = FileTransferManager.generateHash(queryId, path || 'root')
+    const contentType = detectContentType(data)
+    fileTransfer.registerDownload(hash, data, contentType, hostId)
+
+    return {
+      processedData: { [`__ape_link__`]: hash },
+      binaryEntries: [{ path, hash }]
+    }
+  }
+
+  if (Array.isArray(data)) {
+    const processedArray = []
+    const allBinaryEntries = []
+
+    for (let i = 0; i < data.length; i++) {
+      const itemPath = path ? `${path}.${i}` : String(i)
+      const { processedData, binaryEntries } = processBinaryData(
+        data[i], queryId, fileTransfer, hostId, itemPath
+      )
+      processedArray.push(processedData)
+      allBinaryEntries.push(...binaryEntries)
+    }
+
+    return { processedData: processedArray, binaryEntries: allBinaryEntries }
+  }
+
+  if (typeof data === 'object') {
+    const processedObj = {}
+    const allBinaryEntries = []
+
+    for (const key of Object.keys(data)) {
+      const itemPath = path ? `${path}.${key}` : key
+      const { processedData, binaryEntries } = processBinaryData(
+        data[key], queryId, fileTransfer, hostId, itemPath
+      )
+
+      // If this was binary data, mark the key with <!L> tag
+      if (binaryEntries.length > 0 && processedData?.__ape_link__) {
+        processedObj[`${key}<!L>`] = processedData.__ape_link__
+      } else {
+        processedObj[key] = processedData
+      }
+      allBinaryEntries.push(...binaryEntries)
+    }
+
+    return { processedData: processedObj, binaryEntries: allBinaryEntries }
+  }
+
+  // Primitive value - return as-is
+  return { processedData: data, binaryEntries: [] }
+}
+
+module.exports = function sendHandler({ socket, events, hostId, fileTransfer }) {
 
   return function send(queryId, type, data, err) {
     if (!type && !queryId) {
@@ -43,11 +125,24 @@ module.exports = function sendHandler({ socket, events, hostId }) {
       }
       return;
     }
+
+    // Process binary data if fileTransfer is available
+    let processedData = data
+    if (fileTransfer && data && !err) {
+      const { processedData: processed, binaryEntries } = processBinaryData(
+        data, queryId || type, fileTransfer, hostId
+      )
+      processedData = processed
+      if (binaryEntries.length > 0) {
+        console.log(`📦 Registered ${binaryEntries.length} binary download(s) for ${queryId || type}`)
+      }
+    }
+
     if (err) {
       socket.send(jss.stringify({ err: err.message || err, type, queryId }))
       if (typeof onFinish === 'function') onFinish(err, true)
     } else {
-      socket.send(jss.stringify({ data, type, queryId }))
+      socket.send(jss.stringify({ data: processedData, type, queryId }))
       if (typeof onFinish === 'function') onFinish(false, data)
     }