braid-blob 0.0.52 → 0.0.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -47
- package/client.js +79 -0
- package/index.html +146 -0
- package/index.js +107 -43
- package/package.json +3 -2
- package/server-demo.js +22 -4
- package/test/tests.js +21 -23
package/README.md
CHANGED
|
@@ -20,39 +20,27 @@ require('http').createServer((req, res) => {
|
|
|
20
20
|
}).listen(8888)
|
|
21
21
|
```
|
|
22
22
|
|
|
23
|
-
That's it! You now have a blob synchronization server.
|
|
23
|
+
That's it! You now have a blob synchronization server. Upload an image:
|
|
24
24
|
|
|
25
|
-
### Usage Examples
|
|
26
|
-
|
|
27
|
-
First let's upload a file:
|
|
28
25
|
```bash
|
|
29
|
-
curl -X PUT -H "Content-Type: image/png" -T blob.png http://localhost:8888/
|
|
26
|
+
curl -X PUT -H "Content-Type: image/png" -T blob.png http://localhost:8888/blob.png
|
|
30
27
|
```
|
|
31
28
|
|
|
32
|
-
|
|
29
|
+
Then view it at http://localhost:8888/blob.png
|
|
33
30
|
|
|
34
|
-
|
|
31
|
+
### Demo
|
|
35
32
|
|
|
36
|
-
|
|
37
|
-
curl -X PUT -H "Content-Type: text/plain" -d "hello" http://localhost:8888/text
|
|
38
|
-
```
|
|
39
|
-
|
|
40
|
-
Next, subscribe for updates:
|
|
41
|
-
```
|
|
42
|
-
curl -H "Subscribe: true" http://localhost:8888/text
|
|
43
|
-
```
|
|
33
|
+
Run the demo server:
|
|
44
34
|
|
|
45
|
-
Now, in another terminal, write over the file:
|
|
46
35
|
```bash
|
|
47
|
-
|
|
36
|
+
node server-demo.js
|
|
48
37
|
```
|
|
49
38
|
|
|
50
|
-
|
|
39
|
+
Then open http://localhost:8888 in your browser. You can drag and drop images to upload them, and open multiple browser windows to see real-time sync in action.
|
|
51
40
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
```
|
|
41
|
+
<!-- TODO: Add demo video
|
|
42
|
+

|
|
43
|
+
-->
|
|
56
44
|
|
|
57
45
|
## API
|
|
58
46
|
|
|
@@ -61,16 +49,8 @@ curl -X DELETE http://localhost:8888/text
|
|
|
61
49
|
```javascript
|
|
62
50
|
var braid_blob = require('braid-blob')
|
|
63
51
|
|
|
64
|
-
// Set custom
|
|
65
|
-
|
|
66
|
-
braid_blob.db_folder = './custom_files_folder'
|
|
67
|
-
|
|
68
|
-
// Set custom metadata storage location (default: './braid-blob-meta')
|
|
69
|
-
// Stores version metadata and peer information
|
|
70
|
-
braid_blob.meta_folder = './custom_meta_folder'
|
|
71
|
-
|
|
72
|
-
// Set custom peer ID (default: auto-generated and persisted)
|
|
73
|
-
braid_blob.peer = 'my-server-id'
|
|
52
|
+
// Set custom storage location (default: './braid-blobs')
|
|
53
|
+
braid_blob.db_folder = './my-blobs'
|
|
74
54
|
```
|
|
75
55
|
|
|
76
56
|
### `braid_blob.serve(req, res, options)`
|
|
@@ -88,6 +68,21 @@ Handles HTTP requests for blob storage and synchronization.
|
|
|
88
68
|
- `PUT` - Store/update a blob
|
|
89
69
|
- `DELETE` - Remove a blob
|
|
90
70
|
|
|
71
|
+
### `braid_blob.sync(key, url, options)`
|
|
72
|
+
|
|
73
|
+
Bidirectionally synchronizes a blob between local storage and a remote URL.
|
|
74
|
+
|
|
75
|
+
**Parameters:**
|
|
76
|
+
- `key` - Local storage key (string)
|
|
77
|
+
- `url` - Remote URL (URL object)
|
|
78
|
+
- `options` - Optional configuration object
|
|
79
|
+
- `signal` - AbortSignal for cancellation (use to stop sync)
|
|
80
|
+
- `content_type` / `accept` - Content type for requests
|
|
81
|
+
- `on_pre_connect` - Async callback before connection attempt
|
|
82
|
+
- `on_disconnect` - Callback when connection drops
|
|
83
|
+
- `on_unauthorized` - Callback on 401/403 responses
|
|
84
|
+
- `on_res` - Callback receiving the response object
|
|
85
|
+
|
|
91
86
|
### `braid_blob.get(key, options)`
|
|
92
87
|
|
|
93
88
|
Retrieves a blob from local storage or a remote URL.
|
|
@@ -113,30 +108,63 @@ Stores a blob to local storage or a remote URL.
|
|
|
113
108
|
- `body` - Buffer or data to store
|
|
114
109
|
- `options` - Optional configuration object
|
|
115
110
|
- `version` - Version identifier
|
|
116
|
-
- `content_type`
|
|
111
|
+
- `content_type` - Content type of the blob
|
|
117
112
|
- `signal` - AbortSignal for cancellation
|
|
118
113
|
|
|
119
|
-
### `braid_blob.
|
|
114
|
+
### `braid_blob.delete(key, options)`
|
|
120
115
|
|
|
121
|
-
|
|
116
|
+
Deletes a blob from local storage or a remote URL.
|
|
122
117
|
|
|
123
118
|
**Parameters:**
|
|
124
|
-
- `
|
|
125
|
-
- `b` - Second endpoint (local key or URL)
|
|
119
|
+
- `key` - Local storage key (string) or remote URL (URL object)
|
|
126
120
|
- `options` - Optional configuration object
|
|
127
|
-
- `signal` - AbortSignal for cancellation
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
121
|
+
- `signal` - AbortSignal for cancellation
|
|
122
|
+
|
|
123
|
+
## Browser Client
|
|
124
|
+
|
|
125
|
+
A simple browser client is included for subscribing to blob updates.
|
|
126
|
+
|
|
127
|
+
```html
|
|
128
|
+
<script src="https://unpkg.com/braid-http@~1.3/braid-http-client.js"></script>
|
|
129
|
+
<script src="/client.js"></script>
|
|
130
|
+
<script>
|
|
131
|
+
braid_blob_client('http://localhost:8888/blob.png', {
|
|
132
|
+
on_update: (blob, content_type, version) => {
|
|
133
|
+
// Called whenever the blob is updated
|
|
134
|
+
var url = URL.createObjectURL(new Blob([blob], { type: content_type }))
|
|
135
|
+
document.getElementById('image').src = url
|
|
136
|
+
},
|
|
137
|
+
on_delete: () => console.log('Blob was deleted'),
|
|
138
|
+
on_error: (e) => console.error('Error:', e)
|
|
139
|
+
})
|
|
140
|
+
</script>
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### `braid_blob_client(url, options)`
|
|
144
|
+
|
|
145
|
+
Subscribes to a blob endpoint and receives updates.
|
|
146
|
+
|
|
147
|
+
**Parameters:**
|
|
148
|
+
- `url` - The blob endpoint URL
|
|
149
|
+
- `options` - Configuration object
|
|
150
|
+
- `on_update(blob, content_type, version)` - Callback for updates
|
|
151
|
+
- `on_delete` - Callback when blob is deleted
|
|
152
|
+
- `on_error` - Callback for errors
|
|
132
153
|
- `on_res` - Callback receiving the response object
|
|
133
154
|
|
|
155
|
+
**Returns:** `{ stop }` - Call `stop()` to unsubscribe.
|
|
156
|
+
|
|
134
157
|
## Testing
|
|
135
158
|
|
|
136
|
-
|
|
137
|
-
|
|
159
|
+
```bash
|
|
160
|
+
npm install
|
|
161
|
+
node test/test.js
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Or run tests in the browser:
|
|
138
165
|
|
|
139
|
-
|
|
140
|
-
|
|
166
|
+
```bash
|
|
167
|
+
node test/test.js -b
|
|
168
|
+
```
|
|
141
169
|
|
|
142
|
-
|
|
170
|
+
Then open http://localhost:8889/test.html
|
package/client.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Braid-Blob Client
|
|
2
|
+
// requires braid-http@~1.3/braid-http-client.js
|
|
3
|
+
//
|
|
4
|
+
// Usage:
|
|
5
|
+
// braid_blob_client(url, {
|
|
6
|
+
// on_update: (blob, content_type, version) => {
|
|
7
|
+
// // Called whenever there's a new version of the blob
|
|
8
|
+
// console.log('New blob:', blob, content_type, version)
|
|
9
|
+
// }
|
|
10
|
+
// })
|
|
11
|
+
//
|
|
12
|
+
// Returns: { stop } - call stop() to unsubscribe
|
|
13
|
+
//
|
|
14
|
+
function braid_blob_client(url, options = {}) {
|
|
15
|
+
var ac = new AbortController()
|
|
16
|
+
var peer = Math.random().toString(36).substr(2)
|
|
17
|
+
var current_version = null
|
|
18
|
+
|
|
19
|
+
braid_fetch(url, {
|
|
20
|
+
headers: { "Merge-Type": "aww" },
|
|
21
|
+
subscribe: true,
|
|
22
|
+
retry: () => true,
|
|
23
|
+
peer,
|
|
24
|
+
signal: ac.signal
|
|
25
|
+
}).then(res => {
|
|
26
|
+
if (options.on_res) options.on_res(res)
|
|
27
|
+
|
|
28
|
+
res.subscribe(async update => {
|
|
29
|
+
if (update.status === 404) {
|
|
30
|
+
current_version = null
|
|
31
|
+
if (options.on_delete) options.on_delete()
|
|
32
|
+
return
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
var content_type = update.extra_headers?.['content-type']
|
|
36
|
+
var version = update.version?.[0]
|
|
37
|
+
|
|
38
|
+
// Only update if version is newer
|
|
39
|
+
if (compare_events(version, current_version) > 0) {
|
|
40
|
+
current_version = version
|
|
41
|
+
if (options.on_update) options.on_update(update.body, content_type, update.version)
|
|
42
|
+
}
|
|
43
|
+
}, options.on_error || (e => console.error('braid_blob_client error:', e)))
|
|
44
|
+
}).catch(options.on_error || (e => console.error('braid_blob_client error:', e)))
|
|
45
|
+
|
|
46
|
+
return {
|
|
47
|
+
stop: () => ac.abort()
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function compare_events(a, b) {
|
|
51
|
+
if (!a) a = ''
|
|
52
|
+
if (!b) b = ''
|
|
53
|
+
|
|
54
|
+
var c = compare_seqs(get_event_seq(a), get_event_seq(b))
|
|
55
|
+
if (c) return c
|
|
56
|
+
|
|
57
|
+
if (a < b) return -1
|
|
58
|
+
if (a > b) return 1
|
|
59
|
+
return 0
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function get_event_seq(e) {
|
|
63
|
+
if (!e) return ''
|
|
64
|
+
|
|
65
|
+
for (let i = e.length - 1; i >= 0; i--)
|
|
66
|
+
if (e[i] === '-') return e.slice(i + 1)
|
|
67
|
+
return e
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function compare_seqs(a, b) {
|
|
71
|
+
if (!a) a = ''
|
|
72
|
+
if (!b) b = ''
|
|
73
|
+
|
|
74
|
+
if (a.length !== b.length) return a.length - b.length
|
|
75
|
+
if (a < b) return -1
|
|
76
|
+
if (a > b) return 1
|
|
77
|
+
return 0
|
|
78
|
+
}
|
|
79
|
+
}
|
package/index.html
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
<body style="margin: 0; padding: 20px; font-family: sans-serif;">
|
|
2
|
+
<div id="drop_overlay" style="display: none; position: fixed; top: 0; left: 0; right: 0; bottom: 0; background: rgba(0, 100, 200, 0.8); z-index: 1000; justify-content: center; align-items: center;">
|
|
3
|
+
<div style="color: white; font-size: 24px; text-align: center;">
|
|
4
|
+
<div style="font-size: 48px; margin-bottom: 20px;">Drop file here</div>
|
|
5
|
+
<div>to upload to <span id="drop_url"></span></div>
|
|
6
|
+
</div>
|
|
7
|
+
</div>
|
|
8
|
+
<h2>Braid Blob Client Demo</h2>
|
|
9
|
+
<p>
|
|
10
|
+
<label>URL: <input type="text" id="url_input" style="width: 400px;" /></label>
|
|
11
|
+
</p>
|
|
12
|
+
<p>Version: <code id="version_display">-</code></p>
|
|
13
|
+
<p>Content-Type: <code id="content_type_display">-</code></p>
|
|
14
|
+
<div id="image_container">
|
|
15
|
+
<img id="blob_image" style="max-width: 100%; border: 1px solid #ccc;" />
|
|
16
|
+
</div>
|
|
17
|
+
<p id="status">Enter a URL and click Connect</p>
|
|
18
|
+
</body>
|
|
19
|
+
<script src="https://unpkg.com/braid-http@~1.3/braid-http-client.js"></script>
|
|
20
|
+
<script src="/client.js"></script>
|
|
21
|
+
<script>
|
|
22
|
+
var current_object_url = null
|
|
23
|
+
var current_client = null
|
|
24
|
+
var current_blob_url = null
|
|
25
|
+
|
|
26
|
+
// Initialize URL input from query parameter, or default to /test_blob
|
|
27
|
+
var params = new URLSearchParams(location.search)
|
|
28
|
+
var initial_url = params.get('url') || `${location.origin}/blob.png`
|
|
29
|
+
document.getElementById('url_input').value = initial_url
|
|
30
|
+
|
|
31
|
+
function connect(blob_url) {
|
|
32
|
+
current_blob_url = blob_url
|
|
33
|
+
// Stop existing subscription
|
|
34
|
+
if (current_client) {
|
|
35
|
+
current_client.stop()
|
|
36
|
+
current_client = null
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Clear previous image
|
|
40
|
+
if (current_object_url) {
|
|
41
|
+
URL.revokeObjectURL(current_object_url)
|
|
42
|
+
current_object_url = null
|
|
43
|
+
}
|
|
44
|
+
document.getElementById('blob_image').src = ''
|
|
45
|
+
document.getElementById('version_display').textContent = '-'
|
|
46
|
+
document.getElementById('content_type_display').textContent = '-'
|
|
47
|
+
document.getElementById('status').textContent = 'Connecting...'
|
|
48
|
+
|
|
49
|
+
current_client = braid_blob_client(blob_url, {
|
|
50
|
+
on_update: (blob, content_type, version) => {
|
|
51
|
+
document.getElementById('status').textContent = ''
|
|
52
|
+
document.getElementById('version_display').textContent =
|
|
53
|
+
version ? JSON.stringify(version) : '-'
|
|
54
|
+
document.getElementById('content_type_display').textContent =
|
|
55
|
+
content_type || '-'
|
|
56
|
+
|
|
57
|
+
// Revoke old object URL to free memory
|
|
58
|
+
if (current_object_url) {
|
|
59
|
+
URL.revokeObjectURL(current_object_url)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Create a blob URL from the binary data and display it
|
|
63
|
+
var image_blob = new Blob([blob], { type: content_type || 'image/png' })
|
|
64
|
+
current_object_url = URL.createObjectURL(image_blob)
|
|
65
|
+
document.getElementById('blob_image').src = current_object_url
|
|
66
|
+
},
|
|
67
|
+
on_error: (e) => {
|
|
68
|
+
document.getElementById('status').textContent = 'Connecting...'
|
|
69
|
+
},
|
|
70
|
+
on_delete: () => {
|
|
71
|
+
document.getElementById('status').textContent = 'Blob deleted'
|
|
72
|
+
document.getElementById('blob_image').src = ''
|
|
73
|
+
if (current_object_url) {
|
|
74
|
+
URL.revokeObjectURL(current_object_url)
|
|
75
|
+
current_object_url = null
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
})
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
document.getElementById('url_input').onkeydown = (e) => {
|
|
82
|
+
if (e.key === 'Enter') {
|
|
83
|
+
var url = document.getElementById('url_input').value.trim()
|
|
84
|
+
if (url) connect(url)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// Auto-connect on load
|
|
89
|
+
connect(initial_url)
|
|
90
|
+
|
|
91
|
+
// Drag and drop handling
|
|
92
|
+
var drop_overlay = document.getElementById('drop_overlay')
|
|
93
|
+
var drag_counter = 0
|
|
94
|
+
|
|
95
|
+
document.addEventListener('dragenter', (e) => {
|
|
96
|
+
e.preventDefault()
|
|
97
|
+
drag_counter++
|
|
98
|
+
if (drag_counter === 1) {
|
|
99
|
+
document.getElementById('drop_url').textContent = current_blob_url || 'the current URL'
|
|
100
|
+
drop_overlay.style.display = 'flex'
|
|
101
|
+
}
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
document.addEventListener('dragleave', (e) => {
|
|
105
|
+
e.preventDefault()
|
|
106
|
+
drag_counter--
|
|
107
|
+
if (drag_counter === 0) {
|
|
108
|
+
drop_overlay.style.display = 'none'
|
|
109
|
+
}
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
document.addEventListener('dragover', (e) => {
|
|
113
|
+
e.preventDefault()
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
document.addEventListener('drop', async (e) => {
|
|
117
|
+
e.preventDefault()
|
|
118
|
+
drag_counter = 0
|
|
119
|
+
drop_overlay.style.display = 'none'
|
|
120
|
+
|
|
121
|
+
if (!current_blob_url) {
|
|
122
|
+
alert('Please connect to a URL first')
|
|
123
|
+
return
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
var file = e.dataTransfer.files[0]
|
|
127
|
+
if (!file) return
|
|
128
|
+
|
|
129
|
+
document.getElementById('status').textContent = 'Uploading...'
|
|
130
|
+
|
|
131
|
+
try {
|
|
132
|
+
var response = await fetch(current_blob_url, {
|
|
133
|
+
method: 'PUT',
|
|
134
|
+
headers: { 'Content-Type': file.type || 'application/octet-stream' },
|
|
135
|
+
body: file
|
|
136
|
+
})
|
|
137
|
+
if (!response.ok) {
|
|
138
|
+
document.getElementById('status').textContent = 'Upload failed: ' + response.status
|
|
139
|
+
}
|
|
140
|
+
// The subscription will automatically show the new image
|
|
141
|
+
} catch (err) {
|
|
142
|
+
document.getElementById('status').textContent = 'Upload error: ' + err.message
|
|
143
|
+
console.error('Upload error:', err)
|
|
144
|
+
}
|
|
145
|
+
})
|
|
146
|
+
</script>
|
package/index.js
CHANGED
|
@@ -2,18 +2,17 @@ var {http_server: braidify, fetch: braid_fetch} = require('braid-http')
|
|
|
2
2
|
|
|
3
3
|
function create_braid_blob() {
|
|
4
4
|
var braid_blob = {
|
|
5
|
-
db_folder: './braid-
|
|
6
|
-
meta_folder: './braid-
|
|
5
|
+
db_folder: null, // defaults to './braid-blobs'
|
|
6
|
+
meta_folder: null, // defaults to './braid-blobs'
|
|
7
|
+
temp_folder: null, // defaults to './braid-blobs'
|
|
7
8
|
cache: {},
|
|
8
|
-
meta_cache: {},
|
|
9
9
|
key_to_subs: {},
|
|
10
10
|
peer: null, // will be auto-generated if not set by the user
|
|
11
11
|
db: null, // object with read/write/delete methods
|
|
12
|
+
meta_db: null, // sqlite database for meta storage
|
|
12
13
|
reconnect_delay_ms: 1000,
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
var temp_folder = null // will be set in init
|
|
16
|
-
|
|
17
16
|
braid_blob.sync = (a, b, options = {}) => {
|
|
18
17
|
options = normalize_options(options)
|
|
19
18
|
if (!options.peer) options.peer = Math.random().toString(36).slice(2)
|
|
@@ -370,7 +369,7 @@ function create_braid_blob() {
|
|
|
370
369
|
if (options.content_type)
|
|
371
370
|
meta.content_type = options.content_type
|
|
372
371
|
|
|
373
|
-
|
|
372
|
+
save_meta(key, meta)
|
|
374
373
|
if (options.signal?.aborted) return
|
|
375
374
|
|
|
376
375
|
// Notify all subscriptions of the update
|
|
@@ -441,17 +440,85 @@ function create_braid_blob() {
|
|
|
441
440
|
await braid_blob.init()
|
|
442
441
|
|
|
443
442
|
async function real_init() {
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
443
|
+
var fs = require('fs')
|
|
444
|
+
|
|
445
|
+
var db_was_not_set = !braid_blob.db_folder
|
|
446
|
+
if (db_was_not_set)
|
|
447
|
+
braid_blob.db_folder = './braid-blobs'
|
|
448
|
+
|
|
449
|
+
var get_db_folder = () =>
|
|
450
|
+
((typeof braid_blob.db_folder === 'string') &&
|
|
451
|
+
braid_blob.db_folder) || './braid-blobs'
|
|
452
|
+
|
|
453
|
+
// deal with temp folder
|
|
454
|
+
if (!braid_blob.temp_folder) {
|
|
455
|
+
// Deal with versions before 0.0.53
|
|
456
|
+
await fs.promises.rm(
|
|
457
|
+
`${braid_blob.meta_folder || './braid-blob-meta'}/temp`,
|
|
458
|
+
{ recursive: true, force: true })
|
|
459
|
+
|
|
460
|
+
braid_blob.temp_folder = braid_blob.meta_folder ||
|
|
461
|
+
get_db_folder()
|
|
462
|
+
}
|
|
463
|
+
await fs.promises.mkdir(braid_blob.temp_folder,
|
|
464
|
+
{ recursive: true })
|
|
465
|
+
for (var f of await fs.promises.readdir(braid_blob.temp_folder))
|
|
466
|
+
if (f.match(/^temp_\w+$/))
|
|
467
|
+
await fs.promises.unlink(`${braid_blob.temp_folder}/${f}`)
|
|
468
|
+
|
|
469
|
+
// deal with meta folder
|
|
470
|
+
var meta_was_not_set = !braid_blob.meta_folder
|
|
471
|
+
if (meta_was_not_set)
|
|
472
|
+
braid_blob.meta_folder = get_db_folder()
|
|
473
|
+
await fs.promises.mkdir(braid_blob.meta_folder,
|
|
474
|
+
{ recursive: true })
|
|
475
|
+
|
|
476
|
+
// set up sqlite for meta storage
|
|
477
|
+
var Database = require('better-sqlite3')
|
|
478
|
+
braid_blob.meta_db = new Database(
|
|
479
|
+
`${braid_blob.meta_folder}/meta.sqlite`)
|
|
480
|
+
braid_blob.meta_db.pragma('journal_mode = WAL')
|
|
481
|
+
braid_blob.meta_db.exec(`
|
|
482
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
483
|
+
key TEXT PRIMARY KEY,
|
|
484
|
+
value JSON
|
|
485
|
+
)
|
|
486
|
+
`)
|
|
487
|
+
|
|
488
|
+
// Deal with versions before 0.0.53
|
|
489
|
+
async function migrate_meta_files(dir) {
|
|
490
|
+
for (var f of await fs.promises.readdir(dir)) {
|
|
491
|
+
if (!f.match(/\.[0-9a-f]+$/i)) continue
|
|
492
|
+
var key = decode_filename(f)
|
|
493
|
+
var value = JSON.parse(
|
|
494
|
+
await fs.promises.readFile(`${dir}/${f}`, 'utf8'))
|
|
495
|
+
save_meta(key, value)
|
|
496
|
+
await fs.promises.unlink(`${dir}/${f}`)
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
if (meta_was_not_set) {
|
|
500
|
+
try {
|
|
501
|
+
await fs.promises.access('./braid-blob-meta')
|
|
502
|
+
await migrate_meta_files('./braid-blob-meta')
|
|
503
|
+
await fs.promises.rm('./braid-blob-meta', { recursive: true })
|
|
504
|
+
} catch (e) {}
|
|
505
|
+
} else if (braid_blob.meta_folder !== braid_blob.db_folder)
|
|
506
|
+
await migrate_meta_files(braid_blob.meta_folder)
|
|
507
|
+
|
|
508
|
+
// Deal with versions before 0.0.53: migrate db files from ./braid-blob-db
|
|
509
|
+
if (db_was_not_set) {
|
|
510
|
+
try {
|
|
511
|
+
await fs.promises.access('./braid-blob-db')
|
|
512
|
+
for (var f of await fs.promises.readdir('./braid-blob-db')) {
|
|
513
|
+
if (!f.match(/\.[0-9a-f]+$/i)) continue
|
|
514
|
+
await fs.promises.copyFile(
|
|
515
|
+
`./braid-blob-db/${f}`,
|
|
516
|
+
`${braid_blob.db_folder}/${f}`)
|
|
517
|
+
await fs.promises.unlink(`./braid-blob-db/${f}`)
|
|
518
|
+
}
|
|
519
|
+
await fs.promises.rm('./braid-blob-db', { recursive: true })
|
|
520
|
+
} catch (e) {}
|
|
521
|
+
}
|
|
455
522
|
|
|
456
523
|
// Set up db - either use provided object or create file-based storage
|
|
457
524
|
if (typeof braid_blob.db_folder === 'string') {
|
|
@@ -468,7 +535,7 @@ function create_braid_blob() {
|
|
|
468
535
|
},
|
|
469
536
|
write: async (key, data) => {
|
|
470
537
|
var file_path = `${braid_blob.db_folder}/${encode_filename(key)}`
|
|
471
|
-
await atomic_write(file_path, data, temp_folder)
|
|
538
|
+
await atomic_write(file_path, data, braid_blob.temp_folder)
|
|
472
539
|
},
|
|
473
540
|
delete: async (key) => {
|
|
474
541
|
var file_path = `${braid_blob.db_folder}/${encode_filename(key)}`
|
|
@@ -490,34 +557,20 @@ function create_braid_blob() {
|
|
|
490
557
|
}
|
|
491
558
|
}
|
|
492
559
|
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
await require('fs').promises.readFile(
|
|
498
|
-
`${braid_blob.meta_folder}/${encode_filename(key)}`, 'utf8'))
|
|
499
|
-
} catch (e) {
|
|
500
|
-
if (e.code === 'ENOENT')
|
|
501
|
-
braid_blob.meta_cache[key] = {}
|
|
502
|
-
else throw e
|
|
503
|
-
}
|
|
504
|
-
}
|
|
505
|
-
return braid_blob.meta_cache[key]
|
|
560
|
+
function get_meta(key) {
|
|
561
|
+
var row = braid_blob.meta_db.prepare(
|
|
562
|
+
`SELECT value FROM meta WHERE key = ?`).get(key)
|
|
563
|
+
return row ? JSON.parse(row.value) : {}
|
|
506
564
|
}
|
|
507
565
|
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
566
|
+
function save_meta(key, meta) {
|
|
567
|
+
braid_blob.meta_db.prepare(
|
|
568
|
+
`INSERT OR REPLACE INTO meta (key, value) VALUES (?, json(?))`)
|
|
569
|
+
.run(key, JSON.stringify(meta))
|
|
511
570
|
}
|
|
512
571
|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
try {
|
|
516
|
-
await require('fs').promises.unlink(
|
|
517
|
-
`${braid_blob.meta_folder}/${encode_filename(key)}`)
|
|
518
|
-
} catch (e) {
|
|
519
|
-
if (e.code !== 'ENOENT') throw e
|
|
520
|
-
}
|
|
572
|
+
function delete_meta(key) {
|
|
573
|
+
braid_blob.meta_db.prepare(`DELETE FROM meta WHERE key = ?`).run(key)
|
|
521
574
|
}
|
|
522
575
|
|
|
523
576
|
//////////////////////////////////////////////////////////////////
|
|
@@ -663,6 +716,16 @@ function create_braid_blob() {
|
|
|
663
716
|
}
|
|
664
717
|
}
|
|
665
718
|
|
|
719
|
+
function decode_filename(s) {
|
|
720
|
+
// Remove the postfix '.XXX'
|
|
721
|
+
s = s.replace(/\.[^.]+$/, '')
|
|
722
|
+
// Decode percent-encoded characters
|
|
723
|
+
s = decodeURIComponent(s)
|
|
724
|
+
// Swap ! and / (reverse of encode)
|
|
725
|
+
s = s.replace(/[\/!]/g, x => x === '!' ? '/' : '!')
|
|
726
|
+
return s
|
|
727
|
+
}
|
|
728
|
+
|
|
666
729
|
function normalize_options(options = {}) {
|
|
667
730
|
if (!normalize_options.special) {
|
|
668
731
|
normalize_options.special = {
|
|
@@ -704,7 +767,7 @@ function create_braid_blob() {
|
|
|
704
767
|
}
|
|
705
768
|
|
|
706
769
|
async function atomic_write(final_destination, data, temp_folder) {
|
|
707
|
-
var temp = `${temp_folder}
|
|
770
|
+
var temp = `${temp_folder}/temp_${Math.random().toString(36).slice(2)}`
|
|
708
771
|
await require('fs').promises.writeFile(temp, data)
|
|
709
772
|
await require('fs').promises.rename(temp, final_destination)
|
|
710
773
|
}
|
|
@@ -742,6 +805,7 @@ function create_braid_blob() {
|
|
|
742
805
|
|
|
743
806
|
braid_blob.create_braid_blob = create_braid_blob
|
|
744
807
|
braid_blob.braid_fetch = braid_fetch
|
|
808
|
+
braid_blob.encode_filename = encode_filename
|
|
745
809
|
|
|
746
810
|
return braid_blob
|
|
747
811
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "braid-blob",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.54",
|
|
4
4
|
"description": "Library for collaborative blobs over http using braid.",
|
|
5
5
|
"author": "Braid Working Group",
|
|
6
6
|
"repository": "braid-org/braid-blob",
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"test:browser": "node test/test.js --browser"
|
|
11
11
|
},
|
|
12
12
|
"dependencies": {
|
|
13
|
-
"braid-http": "~1.3.86"
|
|
13
|
+
"braid-http": "~1.3.86",
|
|
14
|
+
"better-sqlite3": "^11.7.0"
|
|
14
15
|
}
|
|
15
16
|
}
|
package/server-demo.js
CHANGED
|
@@ -3,22 +3,40 @@ var port = process.argv[2] || 8888
|
|
|
3
3
|
var braid_blob = require(`${__dirname}/index.js`)
|
|
4
4
|
|
|
5
5
|
// TODO: set a custom storage base
|
|
6
|
-
// (the default is ./braid-
|
|
6
|
+
// (the default is ./braid-blobs)
|
|
7
7
|
//
|
|
8
8
|
// braid_blob.db_folder = './custom_files_folder'
|
|
9
|
-
// braid_blob.meta_folder = './custom_meta_folder'
|
|
10
9
|
|
|
11
10
|
braid_blob.init()
|
|
12
11
|
|
|
13
12
|
var server = require("http").createServer(async (req, res) => {
|
|
14
13
|
console.log(`${req.method} ${req.url}`)
|
|
15
14
|
|
|
15
|
+
var url = req.url.split('?')[0]
|
|
16
|
+
|
|
17
|
+
if (url === '/client.js') {
|
|
18
|
+
res.writeHead(200, {
|
|
19
|
+
"Content-Type": "text/javascript",
|
|
20
|
+
"Cache-Control": "no-cache"
|
|
21
|
+
})
|
|
22
|
+
require("fs").createReadStream(`${__dirname}/client.js`).pipe(res)
|
|
23
|
+
return
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
if (url === '/' || url === '/index.html') {
|
|
27
|
+
res.writeHead(200, {
|
|
28
|
+
"Content-Type": "text/html",
|
|
29
|
+
"Cache-Control": "no-cache"
|
|
30
|
+
})
|
|
31
|
+
require("fs").createReadStream(`${__dirname}/index.html`).pipe(res)
|
|
32
|
+
return
|
|
33
|
+
}
|
|
34
|
+
|
|
16
35
|
braid_blob.serve(req, res)
|
|
17
36
|
})
|
|
18
37
|
|
|
19
38
|
server.listen(port, () => {
|
|
20
|
-
console.log(`server started on
|
|
21
|
-
console.log(`files stored in: ${braid_blob.db_folder}`)
|
|
39
|
+
console.log(`server started on http://localhost:${port}`)
|
|
22
40
|
})
|
|
23
41
|
|
|
24
42
|
// curl -X PUT -H "Content-Type: image/png" --data-binary @blob.png http://localhost:8888/blob.png
|
package/test/tests.js
CHANGED
|
@@ -1390,9 +1390,8 @@ runTest(
|
|
|
1390
1390
|
// Get the file to verify it has the expected version
|
|
1391
1391
|
var result1 = await bb1.get(test_key)
|
|
1392
1392
|
|
|
1393
|
-
//
|
|
1394
|
-
|
|
1395
|
-
var debug_info = 'meta1: ' + JSON.stringify(meta1) + '; '
|
|
1393
|
+
// Close the first instance's db
|
|
1394
|
+
bb1.meta_db.close()
|
|
1396
1395
|
|
|
1397
1396
|
// Wait a bit to ensure file system has settled
|
|
1398
1397
|
await new Promise(resolve => setTimeout(resolve, 100))
|
|
@@ -1406,20 +1405,17 @@ runTest(
|
|
|
1406
1405
|
// Initialize bb2 by doing a get (this triggers init)
|
|
1407
1406
|
var result2 = await bb2.get(test_key)
|
|
1408
1407
|
|
|
1409
|
-
// Check what metadata bb2 sees
|
|
1410
|
-
var meta2 = bb2.meta_cache[test_key]
|
|
1411
|
-
debug_info += 'meta2: ' + JSON.stringify(meta2) + '; '
|
|
1412
|
-
|
|
1413
1408
|
// The version should be the same - no new event ID generated
|
|
1414
1409
|
var versions_match = (result1.version[0] === result2.version[0])
|
|
1415
1410
|
var both_have_expected = (result1.version[0] === 'test-peer-123456')
|
|
1416
1411
|
|
|
1417
1412
|
// Clean up
|
|
1413
|
+
bb2.meta_db.close()
|
|
1418
1414
|
await fs.rm(db_folder, { recursive: true, force: true })
|
|
1419
1415
|
await fs.rm(meta_folder, { recursive: true, force: true })
|
|
1420
1416
|
|
|
1421
1417
|
res.end(versions_match && both_have_expected ? 'true' :
|
|
1422
|
-
'false: v1=' + result1.version[0] + ', v2=' + result2.version[0]
|
|
1418
|
+
'false: v1=' + result1.version[0] + ', v2=' + result2.version[0])
|
|
1423
1419
|
} catch (e) {
|
|
1424
1420
|
// Clean up even on error
|
|
1425
1421
|
await fs.rm(db_folder, { recursive: true, force: true })
|
|
@@ -1789,7 +1785,7 @@ runTest(
|
|
|
1789
1785
|
)
|
|
1790
1786
|
|
|
1791
1787
|
runTest(
|
|
1792
|
-
"test atomic write creates
|
|
1788
|
+
"test atomic write creates temp_folder on init",
|
|
1793
1789
|
async () => {
|
|
1794
1790
|
var r1 = await braid_fetch(`/eval`, {
|
|
1795
1791
|
method: 'POST',
|
|
@@ -1807,15 +1803,14 @@ runTest(
|
|
|
1807
1803
|
// Initialize
|
|
1808
1804
|
await bb.init()
|
|
1809
1805
|
|
|
1810
|
-
// Check that
|
|
1811
|
-
var
|
|
1812
|
-
var stat = await fs.stat(temp_folder)
|
|
1813
|
-
var is_dir = stat.isDirectory()
|
|
1806
|
+
// Check that temp_folder is set to meta_folder (no /temp subdirectory anymore)
|
|
1807
|
+
var temp_folder_correct = bb.temp_folder === meta_folder
|
|
1814
1808
|
|
|
1815
|
-
res.end(
|
|
1809
|
+
res.end(temp_folder_correct ? 'true' : 'temp_folder is ' + bb.temp_folder)
|
|
1816
1810
|
} catch (e) {
|
|
1817
1811
|
res.end('error: ' + e.message)
|
|
1818
1812
|
} finally {
|
|
1813
|
+
bb.meta_db.close()
|
|
1819
1814
|
await fs.rm(db_folder, { recursive: true, force: true })
|
|
1820
1815
|
await fs.rm(meta_folder, { recursive: true, force: true })
|
|
1821
1816
|
}
|
|
@@ -1845,14 +1840,15 @@ runTest(
|
|
|
1845
1840
|
// Do a write
|
|
1846
1841
|
await bb.put('/test-file', Buffer.from('hello'), { version: ['1'] })
|
|
1847
1842
|
|
|
1848
|
-
// Check that
|
|
1849
|
-
var
|
|
1850
|
-
var
|
|
1843
|
+
// Check that no temp_ files remain in temp_folder
|
|
1844
|
+
var files = await fs.readdir(bb.temp_folder)
|
|
1845
|
+
var temp_files = files.filter(f => f.startsWith('temp_'))
|
|
1851
1846
|
|
|
1852
|
-
res.end(
|
|
1847
|
+
res.end(temp_files.length === 0 ? 'true' : 'leftover files: ' + temp_files.join(', '))
|
|
1853
1848
|
} catch (e) {
|
|
1854
1849
|
res.end('error: ' + e.message)
|
|
1855
1850
|
} finally {
|
|
1851
|
+
bb.meta_db.close()
|
|
1856
1852
|
await fs.rm(db_folder, { recursive: true, force: true })
|
|
1857
1853
|
await fs.rm(meta_folder, { recursive: true, force: true })
|
|
1858
1854
|
}
|
|
@@ -1890,6 +1886,7 @@ runTest(
|
|
|
1890
1886
|
} catch (e) {
|
|
1891
1887
|
res.end('error: ' + e.message)
|
|
1892
1888
|
} finally {
|
|
1889
|
+
bb.meta_db.close()
|
|
1893
1890
|
await fs.rm(db_folder, { recursive: true, force: true })
|
|
1894
1891
|
await fs.rm(meta_folder, { recursive: true, force: true })
|
|
1895
1892
|
}
|
|
@@ -1926,15 +1923,16 @@ runTest(
|
|
|
1926
1923
|
var content = result.body.toString()
|
|
1927
1924
|
var version = result.version[0]
|
|
1928
1925
|
|
|
1929
|
-
// Also verify
|
|
1930
|
-
var
|
|
1931
|
-
var
|
|
1926
|
+
// Also verify no temp_ files remain
|
|
1927
|
+
var files = await fs.readdir(bb.temp_folder)
|
|
1928
|
+
var temp_files = files.filter(f => f.startsWith('temp_'))
|
|
1932
1929
|
|
|
1933
|
-
res.end(content === 'write3' && version === '3' &&
|
|
1934
|
-
'content=' + content + ', version=' + version + ', temp_files=' +
|
|
1930
|
+
res.end(content === 'write3' && version === '3' && temp_files.length === 0 ? 'true' :
|
|
1931
|
+
'content=' + content + ', version=' + version + ', temp_files=' + temp_files.length)
|
|
1935
1932
|
} catch (e) {
|
|
1936
1933
|
res.end('error: ' + e.message)
|
|
1937
1934
|
} finally {
|
|
1935
|
+
bb.meta_db.close()
|
|
1938
1936
|
await fs.rm(db_folder, { recursive: true, force: true })
|
|
1939
1937
|
await fs.rm(meta_folder, { recursive: true, force: true })
|
|
1940
1938
|
}
|