@andespindola/brainlink 0.1.0-alpha.10 → 0.1.0-alpha.11

package/README.md

@@ -66,6 +66,7 @@ Markdown is the source of truth. `.brainlink/brainlink.db` is only a rebuildable
 - Full-text, semantic and hybrid retrieval modes.
 - SQLite-backed semantic candidate buckets for larger vaults.
 - Agent namespaces under `agents/<agent-id>/`.
+- S3-compatible bucket vaults through `s3://bucket/prefix` URIs.
 - CLI with machine-readable `--json` output.
 - Short CLI alias: `blink`.
 - Built-in MCP stdio server for agent tool integration.
@@ -256,6 +257,36 @@ http://127.0.0.1:4321
 
 When `--vault` is omitted, commands use the default vault at `$HOME/.brainlink/vault`. Pass `--vault` or configure `vault` in `brainlink.config.json` when you want a custom project-local vault.
 
+## Bucket Vaults
+
+Brainlink can use an S3-compatible bucket as the Markdown source of truth:
+
+```bash
+export AWS_REGION="us-east-1"
+export AWS_ACCESS_KEY_ID="..."
+export AWS_SECRET_ACCESS_KEY="..."
+
+blink add "Architecture" \
+  --vault "s3://my-memory-bucket/brainlink" \
+  --content "Bucket Markdown is the source of truth. #architecture"
+
+blink index --vault "s3://my-memory-bucket/brainlink"
+blink context "architecture" --vault "s3://my-memory-bucket/brainlink"
+```
+
+For Cloudflare R2, MinIO or another S3-compatible endpoint:
+
+```bash
+export BRAINLINK_S3_ENDPOINT="https://<account-id>.r2.cloudflarestorage.com"
+export BRAINLINK_S3_FORCE_PATH_STYLE=1
+```
+
+Bucket vaults mirror Markdown into a local cache under
+`$BRAINLINK_HOME/bucket-cache`. The bucket remains canonical; the local
+`.brainlink/brainlink.db` stays a disposable index. Run `index` after remote
+bucket changes before relying on `search`, `context`, graph or validation
+commands. Watch mode is only supported for local filesystem vaults.
+
 ## Core Model
 
 ```txt
@@ -612,6 +643,12 @@ Set `BRAINLINK_ALLOWED_VAULTS` for external wrappers, including MCP servers, so
 export BRAINLINK_ALLOWED_VAULTS="/absolute/path/to/project-vault,/absolute/path/to/team-vault"
 ```
 
+Bucket vaults can be allowlisted with the same variable:
+
+```bash
+export BRAINLINK_ALLOWED_VAULTS="s3://my-memory-bucket/brainlink"
+```
+
 ## Note Format
 
 Brainlink supports Markdown with optional frontmatter:

package/SECURITY.md

@@ -32,4 +32,16 @@ External tool wrappers, including MCP servers, should set `BRAINLINK_ALLOWED_VAU
 export BRAINLINK_ALLOWED_VAULTS="/absolute/path/to/project-vault"
 ```
 
+For bucket vaults, allowlist the S3 URI prefix:
+
+```bash
+export BRAINLINK_ALLOWED_VAULTS="s3://my-memory-bucket/brainlink"
+```
+
 When the allowlist is set, CLI commands fail if `--vault` points outside the allowed roots.
+
+## Bucket Credentials
+
+Bucket vaults use the standard AWS SDK credential chain. Prefer short-lived,
+least-privilege credentials scoped to the specific bucket prefix used by
+Brainlink. Do not store bucket credentials in Markdown notes.

package/dist/application/watch-vault.js

@@ -1,6 +1,6 @@
 import { watch } from 'node:fs';
 import { indexVault } from './index-vault.js';
-import { resolveVaultPath } from '../infrastructure/file-system-vault.js';
+import { isBucketVaultPath, resolveVaultPath } from '../infrastructure/file-system-vault.js';
 const shouldIgnore = (filename) => {
     if (!filename) {
         return false;
@@ -8,6 +8,9 @@ const shouldIgnore = (filename) => {
     return filename.includes('.brainlink') || !filename.endsWith('.md');
 };
 export const startVaultWatcher = (input) => {
+    if (isBucketVaultPath(input.vaultPath)) {
+        throw new Error('Watch mode is only supported for local filesystem vaults.');
+    }
     const absoluteVaultPath = resolveVaultPath(input.vaultPath);
     const debounceMs = input.debounceMs ?? 350;
     let timeout = null;

package/dist/infrastructure/bucket-vault.js

@@ -0,0 +1,171 @@
+import { GetObjectCommand, ListObjectsV2Command, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { chmod, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { dirname, isAbsolute, join, relative } from 'node:path';
+import { posix } from 'node:path';
+import { getBrainlinkHomePath } from './paths.js';
+const directoryMode = 0o700;
+const fileMode = 0o600;
+const bucketScheme = 's3:';
+const manifestPath = '.brainlink/bucket-manifest.json';
+const excludedSegments = new Set(['.brainlink', '.git', 'node_modules', 'dist']);
+export const isBucketVaultUri = (value) => value.trim().toLowerCase().startsWith('s3://');
+const trimSlashes = (value) => value.replace(/^\/+|\/+$/g, '');
+const normalizePrefix = (value) => trimSlashes(posix.normalize(trimSlashes(value))).replace(/^\.$/, '');
+export const parseBucketVaultUri = (uri) => {
+    const parsed = new URL(uri);
+    if (parsed.protocol !== bucketScheme || !parsed.hostname) {
+        throw new Error(`Unsupported bucket vault URI: ${uri}. Use s3://bucket/prefix.`);
+    }
+    return {
+        uri: formatBucketVaultUri(parsed.hostname, normalizePrefix(decodeURIComponent(parsed.pathname))),
+        bucket: parsed.hostname,
+        prefix: normalizePrefix(decodeURIComponent(parsed.pathname))
+    };
+};
+export const formatBucketVaultUri = (bucket, prefix) => prefix ? `s3://${bucket}/${prefix}` : `s3://${bucket}`;
+export const getBucketVaultCachePath = (uri) => {
+    const hash = createHash('sha256').update(parseBucketVaultUri(uri).uri).digest('hex').slice(0, 24);
+    return join(getBrainlinkHomePath(), 'bucket-cache', hash);
+};
+const ensureDirectory = async (path) => {
+    await mkdir(path, { recursive: true, mode: directoryMode });
+    await chmod(path, directoryMode);
+};
+const isPathInside = (parent, child) => {
+    const path = relative(parent, child);
+    return path === '' || (!path.startsWith('..') && !isAbsolute(path));
+};
+const toSafeRelativePath = (key) => {
+    const normalized = normalizePrefix(key);
+    if (!normalized || normalized.split('/').some((segment) => segment === '..' || excludedSegments.has(segment))) {
+        return null;
+    }
+    return normalized.endsWith('.md') ? normalized : null;
+};
+const toObjectKey = (reference, relativePath) => reference.prefix ? `${reference.prefix}/${relativePath}` : relativePath;
+const toRelativeObjectKey = (reference, objectKey) => {
+    const relativePath = reference.prefix
+        ? objectKey.startsWith(`${reference.prefix}/`)
+            ? objectKey.slice(reference.prefix.length + 1)
+            : null
+        : objectKey;
+    return relativePath ? toSafeRelativePath(relativePath) : null;
+};
+const createBucketClient = () => new S3Client({
+    region: process.env.AWS_REGION ?? process.env.BRAINLINK_S3_REGION ?? 'us-east-1',
+    endpoint: process.env.BRAINLINK_S3_ENDPOINT ?? process.env.AWS_ENDPOINT_URL,
+    forcePathStyle: process.env.BRAINLINK_S3_FORCE_PATH_STYLE === '1'
+});
+const streamToString = async (body) => {
+    if (body && typeof body === 'object' && 'transformToString' in body && typeof body.transformToString === 'function') {
+        return body.transformToString();
+    }
+    throw new Error('Unsupported S3 object body.');
+};
+const readManifest = async (cachePath) => {
+    try {
+        return JSON.parse(await readFile(join(cachePath, manifestPath), 'utf8'));
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+            return {
+                uri: '',
+                keys: []
+            };
+        }
+        throw error;
+    }
+};
+const writeManifest = async (cachePath, manifest) => {
+    const path = join(cachePath, manifestPath);
+    await ensureDirectory(dirname(path));
+    await writeFile(path, `${JSON.stringify(manifest, null, 2)}\n`, { encoding: 'utf8', mode: fileMode });
+    await chmod(path, fileMode);
+};
+const listBucketMarkdownKeys = async (client, reference) => {
+    const keys = [];
+    let continuationToken;
+    do {
+        const result = await client.send(new ListObjectsV2Command({
+            Bucket: reference.bucket,
+            Prefix: reference.prefix ? `${reference.prefix}/` : undefined,
+            ContinuationToken: continuationToken
+        }));
+        keys.push(...(result.Contents ?? []).flatMap((object) => (object.Key ? [object.Key] : [])));
+        continuationToken = result.NextContinuationToken;
+    } while (continuationToken);
+    return keys.flatMap((key) => {
+        const relativePath = toRelativeObjectKey(reference, key);
+        return relativePath ? [relativePath] : [];
+    });
+};
+const removeStaleCachedFiles = async (cachePath, previousKeys, currentKeys) => {
+    await Promise.all(previousKeys
+        .filter((key) => !currentKeys.has(key))
+        .map(async (key) => {
+        const absolutePath = join(cachePath, key);
+        if (isPathInside(cachePath, absolutePath)) {
+            await rm(absolutePath, { force: true });
+        }
+    }));
+};
+const downloadMarkdownFiles = async (client, reference, cachePath, keys) => {
+    await Promise.all(keys.map(async (key) => {
+        const absolutePath = join(cachePath, key);
+        if (!isPathInside(cachePath, absolutePath)) {
+            throw new Error(`Refusing to cache bucket object outside vault cache: ${key}`);
+        }
+        const result = await client.send(new GetObjectCommand({
+            Bucket: reference.bucket,
+            Key: toObjectKey(reference, key)
+        }));
+        await ensureDirectory(dirname(absolutePath));
+        await writeFile(absolutePath, await streamToString(result.Body), { encoding: 'utf8', mode: fileMode });
+        await chmod(absolutePath, fileMode);
+    }));
+};
+export const syncBucketVaultToCache = async (uri) => {
+    const reference = parseBucketVaultUri(uri);
+    const cachePath = getBucketVaultCachePath(reference.uri);
+    const client = createBucketClient();
+    const previousManifest = await readManifest(cachePath);
+    const keys = await listBucketMarkdownKeys(client, reference);
+    const currentKeys = new Set(keys);
+    await ensureDirectory(join(cachePath, '.brainlink'));
+    await removeStaleCachedFiles(cachePath, previousManifest.uri === reference.uri ? previousManifest.keys : [], currentKeys);
+    await downloadMarkdownFiles(client, reference, cachePath, keys);
+    await writeManifest(cachePath, {
+        uri: reference.uri,
+        keys
+    });
+    return cachePath;
+};
+export const writeBucketMarkdownFile = async (uri, filename, content) => {
+    const reference = parseBucketVaultUri(uri);
+    const cachePath = getBucketVaultCachePath(reference.uri);
+    const relativePath = toSafeRelativePath(filename.endsWith('.md') ? filename : `${filename}.md`);
+    if (!relativePath) {
+        throw new Error(`Invalid bucket Markdown path: ${filename}`);
+    }
+    const absolutePath = join(cachePath, relativePath);
+    if (!isPathInside(cachePath, absolutePath)) {
+        throw new Error(`Refusing to write outside bucket cache: ${absolutePath}`);
+    }
+    await ensureDirectory(join(cachePath, '.brainlink'));
+    await ensureDirectory(dirname(absolutePath));
+    await writeFile(absolutePath, content, { encoding: 'utf8', mode: fileMode });
+    await chmod(absolutePath, fileMode);
+    await createBucketClient().send(new PutObjectCommand({
+        Bucket: reference.bucket,
+        Key: toObjectKey(reference, relativePath),
+        Body: content,
+        ContentType: 'text/markdown; charset=utf-8'
+    }));
+    const manifest = await readManifest(cachePath);
+    await writeManifest(cachePath, {
+        uri: reference.uri,
+        keys: Array.from(new Set([...manifest.keys, relativePath])).sort()
+    });
+    return `${reference.uri}/${relativePath}`;
+};

package/dist/infrastructure/file-system-vault.js

@@ -1,6 +1,7 @@
 import { chmod, mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { dirname, extname, isAbsolute, join, relative, resolve } from 'node:path';
 import { resolvePath } from './paths.js';
+import { getBucketVaultCachePath, isBucketVaultUri, parseBucketVaultUri, syncBucketVaultToCache, writeBucketMarkdownFile } from './bucket-vault.js';
 const excludedDirectories = new Set(['.brainlink', '.git', 'node_modules', 'dist']);
 const directoryMode = 0o700;
 const fileMode = 0o600;
@@ -15,30 +16,44 @@ const walkMarkdownFiles = async (directory) => {
     }));
     return nested.flat();
 };
-export const resolveVaultPath = (vaultPath) => resolvePath(vaultPath);
+export const resolveVaultPath = (vaultPath) => isBucketVaultUri(vaultPath) ? getBucketVaultCachePath(vaultPath) : resolvePath(vaultPath);
+export const isBucketVaultPath = (vaultPath) => isBucketVaultUri(vaultPath);
 const isPathInside = (parent, child) => {
     const path = relative(parent, child);
     return path === '' || (!path.startsWith('..') && !isAbsolute(path));
 };
+const isBucketPrefixInside = (parent, child) => parent === '' || child === parent || child.startsWith(`${parent}/`);
 const secureDirectory = async (path) => {
     await mkdir(path, { recursive: true, mode: directoryMode });
     await chmod(path, directoryMode);
 };
 export const assertVaultAllowed = (vaultPath, allowedVaults) => {
+    if (isBucketVaultUri(vaultPath)) {
+        const vault = parseBucketVaultUri(vaultPath);
+        const allowed = allowedVaults.filter(isBucketVaultUri).map(parseBucketVaultUri);
+        if (allowedVaults.length > 0 &&
+            !allowed.some((allowedVault) => vault.bucket === allowedVault.bucket && isBucketPrefixInside(allowedVault.prefix, vault.prefix))) {
+            throw new Error(`Vault path is not allowed: ${vault.uri}. Configure BRAINLINK_ALLOWED_VAULTS or allowedVaults.`);
+        }
+        return vault.uri;
+    }
     const absoluteVaultPath = resolveVaultPath(vaultPath);
-    const allowed = allowedVaults.map(resolveVaultPath);
+    const allowed = allowedVaults.filter((allowedVault) => !isBucketVaultUri(allowedVault)).map(resolveVaultPath);
     if (allowed.length > 0 && !allowed.some((allowedPath) => isPathInside(allowedPath, absoluteVaultPath))) {
         throw new Error(`Vault path is not allowed: ${absoluteVaultPath}. Configure BRAINLINK_ALLOWED_VAULTS or allowedVaults.`);
     }
     return absoluteVaultPath;
 };
 export const ensureVault = async (vaultPath) => {
+    if (isBucketVaultUri(vaultPath)) {
+        return syncBucketVaultToCache(vaultPath);
+    }
     const absoluteVaultPath = resolveVaultPath(vaultPath);
     await secureDirectory(join(absoluteVaultPath, '.brainlink'));
     return absoluteVaultPath;
 };
 export const readMarkdownFiles = async (vaultPath) => {
-    const absoluteVaultPath = resolveVaultPath(vaultPath);
+    const absoluteVaultPath = await ensureVault(vaultPath);
     const paths = await walkMarkdownFiles(absoluteVaultPath);
     return Promise.all(paths.map(async (absolutePath) => {
         const [content, stats] = await Promise.all([readFile(absolutePath, 'utf8'), stat(absolutePath)]);
@@ -51,6 +66,9 @@ export const readMarkdownFiles = async (vaultPath) => {
     }));
 };
 export const writeMarkdownFile = async (vaultPath, filename, content) => {
+    if (isBucketVaultUri(vaultPath)) {
+        return writeBucketMarkdownFile(vaultPath, filename, content);
+    }
     const absoluteVaultPath = await ensureVault(vaultPath);
     const absolutePath = resolve(absoluteVaultPath, filename.endsWith('.md') ? filename : `${filename}.md`);
     if (!isPathInside(absoluteVaultPath, absolutePath)) {

package/docs/AGENT_USAGE.md

@@ -552,4 +552,5 @@ Weak retrieval usually means:
 - Local embeddings are deterministic and provider-free; remote embedding providers are not implemented yet.
 - MCP integration is available through the `brainlink-mcp` stdio server.
 - HTTP API is local and unauthenticated.
-- Watch mode depends on platform filesystem watcher behavior.
+- Bucket vaults support S3-compatible `s3://bucket/prefix` URIs and use a local cache for SQLite indexes.
+- Watch mode depends on platform filesystem watcher behavior and is only supported for local filesystem vaults.

package/docs/ARCHITECTURE.md

@@ -105,13 +105,15 @@ Application code depends on domain rules and infrastructure interfaces.
 The infrastructure layer handles side effects:
 
 - reading Markdown files from disk
+- mirroring S3-compatible bucket Markdown into a local cache
 - writing Markdown notes
 - creating `.brainlink`
 - writing and querying SQLite
 - running FTS, semantic and hybrid retrieval
 - narrowing semantic candidates through SQLite embedding buckets before cosine scoring
 
-SQLite is an index, not the canonical storage model.
+SQLite is an index, not the canonical storage model. For bucket vaults, Markdown
+objects in the bucket remain canonical and SQLite is still local derived data.
 
 ## Indexing Flow
 
@@ -240,6 +242,7 @@ Relevant content
 Permanent:
 
 - Markdown files
+- S3-compatible Markdown objects when the vault is `s3://bucket/prefix`
 - optional Git history around the vault
 
 Canonical agent memory lives under:
@@ -251,6 +254,7 @@ vault/agents/<agent-id>/**/*.md
 Rebuildable:
 
 - `.brainlink/brainlink.db`
+- `$BRAINLINK_HOME/bucket-cache`
 - FTS records
 - local embedding vectors
 - local embedding bucket index

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@andespindola/brainlink",
-  "version": "0.1.0-alpha.10",
+  "version": "0.1.0-alpha.11",
   "description": "Local-first knowledge memory for agents with Markdown, backlinks, indexing and context retrieval.",
   "type": "module",
   "license": "MIT",
@@ -54,6 +54,7 @@
     "pack:smoke": "npm pack --dry-run"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1038.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "better-sqlite3": "^12.9.0",
     "commander": "^14.0.2",