braidfs 0.0.14 → 0.0.16

package/README.md

@@ -1,67 +1,60 @@
 # braidfs
-braid technology synchronizing files and webpages
+Proxy braid collaborative text pages as editable files on disk.
 
-## Features
-
-- Proxies web resources as collaborative text using the Braid protocol
-- Caches proxied content locally
-- Monitors local files for changes and syncs them back to the origin
-- Supports pinning specific URLs
 
 ## Installation
 
-Clone the repository:
+Install braidfs globally using npm:
 
-```bash
-git clone https://github.com/braid-org/braidfs.git
-cd braidfs
 ```
-
-Install dependencies:
-
-```bash
-npm install
+npm install -g braidfs
 ```
 
 ## Usage
 
-Run the server:
+To start braidfs, run the following command:
 
-```bash
-node index.js [PORT] [OPTIONS]
+```
+braidfs [port] [-pin <url>] [-pin index <url>]
 ```
 
-### Options
-
-- `PORT`: The port number to run the server on (default: 10000)
-- `-pin URL`: Pin a specific URL to be proxied
-- `-pin index URL`: Pin an index URL that contains a list of URLs to be proxied
-- `COOKIE`: Set a cookie to be used in requests (optional)
+- `[port]`: Optional. Specify the port number (default is 10000).
+- `-pin <url>`: Pin a specific URL for synchronization.
+- `-pin index <url>`: Pin an index URL that contains a list of URLs to synchronize.
 
 Example:
 
-```bash
-node index.js 60000 -pin https://example.com/resource -pin index https://example.com/index.json mycookie=value
+```
+braidfs 8080 -pin https://example.com/document.txt -pin index https://example.com/index.json
 ```
 
-This will run a server on port 60000, pin the specified URL, use the index URL to proxy multiple resources, and set a cookie for requests.
+## Configuration
 
-## Accessing the Proxy
+braidfs looks for a configuration file at `~/.braidfs/config.json`. You can set the following options:
 
-The proxy only allows connections from localhost for security reasons.
+- `port`: The port number for the proxy server.
+- `pin_urls`: An array of URLs to pin for synchronization.
+- `pindex_urls`: An array of index URLs containing lists of URLs to synchronize.
+- `proxy_base`: The base directory for storing proxied files (default is `~/http`).
 
-- `/pages`: Shows all the proxied URLs
-- `/URL`: Proxies the specified URL as Braid text and creates a file in `proxy_base/URL`
+Example `config.json`:
 
-## File Structure
+```json
+{
+  "port": 9000,
+  "pin_urls": ["https://example.com/document1.txt", "https://example.com/document2.txt"],
+  "pindex_urls": ["https://example.com/index.json"],
+  "proxy_base": "/path/to/custom/proxy/directory"
+}
+```
 
-- `braid-text-db`: Stores local cached information
-- `proxy_base`: Stores proxied files, which are updated when resources change and monitored for local changes
+## Accessing the Proxy
 
-## Security Note
+The proxy only allows connections from localhost for security reasons.
 
-This proxy is designed to be accessible only from localhost. Attempting to access it from any other IP address will result in a 403 Forbidden error.
+- `/pages`: Shows all the proxied URLs
+- `/URL`: Proxies the specified URL and creates a file in `proxy_base/URL`
 
-## Known Issues
+## Security
 
-- The server doesn't automatically resubscribe to proxied resources when restarted, though it will keep trying to re-establish broken connections while running.
+braidfs is designed to run locally and only accepts connections from localhost (127.0.0.1 or ::1) for security reasons.

package/index.js

@@ -7,34 +7,47 @@ let braid_fetch = require('braid-http').fetch
 process.on("unhandledRejection", (x) => console.log(`unhandledRejection: ${x.stack}`))
 process.on("uncaughtException", (x) => console.log(`uncaughtException: ${x.stack}`))
 
-let port = 10000
-let cookie = null
-let pin_urls = []
-let pindex_urls = []
-let proxy_base = `./proxy_base`
-let proxy_base_support = `./proxy_base_support`
+let braidfs_config_dir = require('path').join(require('os').homedir(), '.braidfs')
+let braidfs_config_path = require('path').join(braidfs_config_dir, 'config.json')
+braid_text.db_folder = require('path').join(braidfs_config_dir, 'braid-text-db')
+
+let config = {
+    port: 10000,
+    pin_urls: [],
+    pindex_urls: [],
+    proxy_base: require('path').join(require('os').homedir(), 'http'),
+    proxy_base_last_versions: require('path').join(braidfs_config_dir, 'proxy_base_last_versions')
+}
+
+// process config file
+try {
+    console.log(`loading config file at: ${braidfs_config_path}`)
+    Object.assign(config, JSON.parse(require('fs').readFileSync(braidfs_config_path, 'utf8')))
+} catch (e) { console.error(`Error loading config file:`, e.message) }
 
+// process command line args (override config)
 let argv = process.argv.slice(2)
 while (argv.length) {
     let a = argv.shift()
     if (a.match(/^\d+$/)) {
-        port = parseInt(a)
+        config.port = parseInt(a)
     } else if (a === '-pin') {
         let b = argv.shift()
         if (b === 'index') {
-            pindex_urls.push(argv.shift())
+            config.pindex_urls.push(argv.shift())
         } else {
-            pin_urls.push(b)
+            config.pin_urls.push(b)
         }
-    } else {
-        cookie = a
-        console.log(`cookie = ${cookie}`)
     }
 }
-console.log({ pin_urls, pindex_urls })
 
-for (let url of pin_urls) proxy_url(url)
-pindex_urls.forEach(async url => {
+// create directories
+require('fs').mkdirSync(config.proxy_base, { recursive: true })
+require('fs').mkdirSync(config.proxy_base_last_versions, { recursive: true })
+
+console.log({ pin_urls: config.pin_urls, pindex_urls: config.pindex_urls })
+for (let url of config.pin_urls) proxy_url(url)
+config.pindex_urls.forEach(async url => {
     let prefix = new URL(url).origin
     while (true) {
         let urls = await (await fetch(url)).json()
@@ -85,16 +98,20 @@ const server = http.createServer(async (req, res) => {
     proxy_url(url)
 
     // Now serve the collaborative text!
-    braid_text.serve(req, res, { key: url })
+    braid_text.serve(req, res, { key: normalize_url(url) })
 });
 
-server.listen(port, () => {
-    console.log(`Proxy server started on port ${port}`);
+server.listen(config.port, () => {
+    console.log(`Proxy server started on port ${config.port}`);
     console.log('This proxy is only accessible from localhost');
 });
 
 ////////////////////////////////
 
+function normalize_url(url) {
+    return url.replace(/(\/index|\/)+$/, '')
+}
+
 async function proxy_url(url) {
     let chain = proxy_url.chain || (proxy_url.chain = Promise.resolve())
 
@@ -115,7 +132,7 @@ async function proxy_url(url) {
                         await require("fs").promises.mkdir(path, { recursive: true })
 
                         while (await is_dir(partial))
-                            partial = require("path").join(partial, 'index.html')
+                            partial = require("path").join(partial, 'index')
 
                         await require("fs").promises.writeFile(partial, save)
                         break
@@ -125,8 +142,8 @@ async function proxy_url(url) {
         }))
     }
 
-    // normalize url by removing any trailing /index.html/index.html/
-    let normalized_url = url.replace(/(\/index\.html|\/)+$/, '')
+    // normalize url by removing any trailing /index/index/
+    let normalized_url = normalize_url(url)
     let wasnt_normal = normalized_url != url
     url = normalized_url
 
@@ -137,19 +154,17 @@ async function proxy_url(url) {
     console.log(`proxy_url: ${url}`)
 
     let path = url.replace(/^https?:\/\//, '')
-    let fullpath = require("path").join(proxy_base, path)
+    let fullpath = require("path").join(config.proxy_base, path)
 
-    // if we're accessing /blah/index.html, it will be normalized to /blah,
+    // if we're accessing /blah/index, it will be normalized to /blah,
     // but we still want to create a directory out of blah in this case
     if (wasnt_normal && !(await is_dir(fullpath))) await ensure_path(fullpath)
 
     await ensure_path(require("path").dirname(fullpath))
 
-    await require("fs").promises.mkdir(proxy_base_support, { recursive: true })
-
     async function get_fullpath() {
         let p = fullpath
-        while (await is_dir(p)) p = require("path").join(p, 'index.html')
+        while (await is_dir(p)) p = require("path").join(p, 'index')
         return p
     }
 
@@ -176,7 +191,7 @@ async function proxy_url(url) {
             headers: {
                 "Merge-Type": "dt",
                 "Content-Type": 'text/plain',
-                ...(cookie ? { "Cookie": cookie } : {}),
+                ...config?.domains?.[(new URL(url)).hostname]?.auth_headers,
             },
             method: "PUT",
             retry: true,
@@ -191,7 +206,7 @@ async function proxy_url(url) {
 
         if (file_last_version === null) {
             try {
-                file_last_version = JSON.parse(await require('fs').promises.readFile(require('path').join(proxy_base_support, braid_text.encode_filename(url)), { encoding: 'utf8' }))
+                file_last_version = JSON.parse(await require('fs').promises.readFile(require('path').join(config.proxy_base_last_versions, braid_text.encode_filename(url)), { encoding: 'utf8' }))
                 file_last_text = (await braid_text.get(url, { version: file_last_version })).body
                 file_needs_writing = !v_eq(file_last_version, (await braid_text.get(url, {})).version)
             } catch (e) {
@@ -223,7 +238,7 @@ async function proxy_url(url) {
 
                     await braid_text.put(url, { version, parents, patches, peer })
 
-                    await require('fs').promises.writeFile(require('path').join(proxy_base_support, braid_text.encode_filename(url)), JSON.stringify(file_last_version))
+                    await require('fs').promises.writeFile(require('path').join(config.proxy_base_last_versions, braid_text.encode_filename(url)), JSON.stringify(file_last_version))
                 }
             }
             if (file_needs_writing) {
@@ -236,7 +251,7 @@ async function proxy_url(url) {
                     file_last_version = version
                     file_last_text = body
                     await require('fs').promises.writeFile(await get_fullpath(), file_last_text)
-                    await require('fs').promises.writeFile(require('path').join(proxy_base_support, braid_text.encode_filename(url)), JSON.stringify(file_last_version))
+                    await require('fs').promises.writeFile(require('path').join(config.proxy_base_last_versions, braid_text.encode_filename(url)), JSON.stringify(file_last_version))
                 }
             }
         }
@@ -271,11 +286,11 @@ async function proxy_url(url) {
 
     if (!proxy_url.chokidar) {
         proxy_url.chokidar = true
-        require('chokidar').watch(proxy_base).on('change', (path) => {
-            path = require('path').relative(proxy_base, path)
+        require('chokidar').watch(config.proxy_base).on('change', (path) => {
+            path = require('path').relative(config.proxy_base, path)
             console.log(`path changed: ${path}`)
 
-            path = path.replace(/(\/index\.html|\/)+$/, '')
+            path = normalize_url(path)
             // console.log(`normalized path: ${path}`)
 
             proxy_url.path_to_func[path]()
@@ -306,7 +321,6 @@ async function proxy_url(url) {
             if (version.length == 0) return;
 
             // console.log(`local got: ${JSON.stringify({ version, parents, body, patches }, null, 4)}`)
-            // console.log(`cookie = ${cookie}`)
 
             signal_file_needs_writing()
 

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "braidfs",
-  "version": "0.0.14",
+  "version": "0.0.16",
   "description": "braid technology synchronizing files and webpages",
   "author": "Braid Working Group",
   "repository": "braid-org/braidfs",
   "homepage": "https://braid.org",
   "dependencies": {
     "braid-http": "^0.3.20",
-    "braid-text": "^0.0.25",
+    "braid-text": "^0.0.26",
     "chokidar": "^3.6.0"
   }
 }

package/~/http/braid.org/braid-text/hi101

@@ -0,0 +1 @@
+abcd