birdclaw 0.9.1 → 0.9.2

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 0.9.2 - 2026-07-04
+
+### Fixed
+
+- Split oversized backup JSONL outputs into deterministic 48 MiB part files and serialize and validate them sequentially, keeping large archives below hosted Git limits without Git LFS.
+
 ## 0.9.1 - 2026-07-02
 
 ### Added

package/README.md

@@ -598,9 +598,11 @@ data/dms/conversations.jsonl
 data/dms/YYYY.jsonl
 data/moderation/blocks.jsonl
 data/moderation/mutes.jsonl
+data/<logical-shard>.part-NNNN.jsonl
 ```
 
 Tweets are sharded by year for human browsing and yearly analysis. Collection-only tweets whose real creation date is unknown go into `data/tweets/unknown.jsonl` instead of pretending they belong to 1970. Timeline membership is stored in `data/timeline_edges`; likes and bookmarks are stored as account-scoped collection edges in `data/collections`. DMs are sharded by year with `conversation_id` in each row; this keeps Git fast while preserving conversation membership.
+Any logical shard larger than 48 MiB is split into deterministic `.part-0001.jsonl` files. Import and validation accept both numbered parts and the earlier unsplit layout, so large private backups remain ordinary Git repositories without Git LFS.
 
 Use `backup sync` when the target is a private Git repo. It pulls first, merge-imports the remote backup into local SQLite, exports the local union back into text shards, commits, and pushes.
 

package/dist/cli/birdclaw.js

@@ -187,6 +187,10 @@ var __test__ = {
 };
 
 // src/lib/backup-table-codecs.ts
+var BACKUP_SHARD_PART_PATTERN = /\.part-\d{4,}\.jsonl$/u;
+function logicalBackupShardPath(relativePath) {
+  return relativePath.replace(BACKUP_SHARD_PART_PATTERN, ".jsonl");
+}
 function fixedShard(relativePath, countKey) {
   return {
     shardPath: () => relativePath,
@@ -1205,7 +1209,8 @@ function adaptLegacyTweetState(schemaVersion, tweets, collections, timelineEdges
   return { collections: nextCollections, timelineEdges: nextTimelineEdges };
 }
 function backupCodecForPath(relativePath, codecs = BACKUP_TABLE_CODECS) {
-  const matches = codecs.filter((codec) => codec.matchesPath(relativePath));
+  const logicalPath = logicalBackupShardPath(relativePath);
+  const matches = codecs.filter((codec) => codec.matchesPath(logicalPath));
   if (matches.length !== 1) {
     throw new Error(
       matches.length === 0 ? `No backup codec owns path: ${relativePath}` : `Multiple backup codecs own path: ${relativePath}`
@@ -1234,7 +1239,7 @@ function countBackupFiles(files, codecs = BACKUP_TABLE_CODECS) {
   for (const file of files) {
     if (!file.path.startsWith("data/")) continue;
     const codec = backupCodecForPath(file.path, codecs);
-    const key = codec.countKey(file.path);
+    const key = codec.countKey(logicalBackupShardPath(file.path));
     counts[key] = (counts[key] ?? 0) + file.rows;
   }
   return counts;
@@ -3716,8 +3721,9 @@ function runSubprocessEffect(options) {
 }
 
 // src/lib/backup.ts
-var BACKUP_SCHEMA_VERSION = 3;
+var BACKUP_SCHEMA_VERSION = 4;
 var MIN_SUPPORTED_BACKUP_SCHEMA_VERSION = 1;
+var DEFAULT_MAX_BACKUP_SHARD_BYTES = 48 * 1024 * 1024;
 var MANIFEST_PATH = "manifest.json";
 var DATA_DIR = "data";
 var GITATTRIBUTES_PATH = ".gitattributes";
@@ -3815,6 +3821,50 @@ function getExportRowSets(db) {
 function buildShards(db) {
   return buildBackupShardsFromRowSets(getExportRowSets(db));
 }
+function normalizeMaxBackupShardBytes(value) {
+  const maxBytes = value ?? DEFAULT_MAX_BACKUP_SHARD_BYTES;
+  if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) {
+    throw new Error("Backup shard byte limit must be a positive integer");
+  }
+  return maxBytes;
+}
+function partPath(relativePath, partNumber) {
+  if (!relativePath.endsWith(".jsonl")) {
+    throw new Error(`Backup shard path must end in .jsonl: ${relativePath}`);
+  }
+  return relativePath.replace(
+    /\.jsonl$/u,
+    `.part-${String(partNumber).padStart(4, "0")}.jsonl`
+  );
+}
+function splitJsonlShard(relativePath, rows, maxBytes) {
+  const parts = [];
+  let currentRows = [];
+  let currentBytes = 0;
+  for (const [index, row] of rows.entries()) {
+    const rowBytes = Buffer.byteLength(jsonlStringify(row)) + 1;
+    if (rowBytes > maxBytes) {
+      throw new Error(
+        `Backup row exceeds shard byte limit: ${relativePath}:${index + 1} is ${rowBytes} bytes (limit ${maxBytes})`
+      );
+    }
+    if (currentRows.length > 0 && currentBytes + rowBytes > maxBytes) {
+      parts.push(currentRows);
+      currentRows = [];
+      currentBytes = 0;
+    }
+    currentRows.push(row);
+    currentBytes += rowBytes;
+  }
+  if (currentRows.length > 0) parts.push(currentRows);
+  if (parts.length <= 1) {
+    return [{ relativePath, rows: parts[0] ?? [] }];
+  }
+  return parts.map((partRows, index) => ({
+    relativePath: partPath(relativePath, index + 1),
+    rows: partRows
+  }));
+}
 function writeJsonlFileEffect(repoPath, relativePath, rows) {
   return Effect5.gen(function* () {
     const fullPath = yield* trySync2(
@@ -3951,6 +4001,7 @@ data/follow_events.jsonl
 \`\`\`
 
 Tweets are sharded by creation year. Collection-only tweets whose creation date is unknown live in \`data/tweets/unknown.jsonl\`. Timeline edges keep account-scoped home/mention membership separate from canonical tweet content. DMs are sharded by year and keep \`conversation_id\` in each row.
+Logical shards larger than 48 MiB are split into deterministic \`.part-0001.jsonl\` files so ordinary Git hosting remains usable without Git LFS.
 The links shard stores expanded short URLs and their source tweet/DM occurrences so linked-tweet search can be rebuilt without re-expanding every \`t.co\` URL.
 
 Never commit live tokens, browser cookies, raw SQLite WAL/SHM sidecars, or temporary cache files here.
@@ -4202,7 +4253,8 @@ function exportBackupEffect({
   commit = false,
   push = false,
   message = "archive: update birdclaw backup",
-  validate = true
+  validate = true,
+  maxShardBytes
 }) {
   return Effect5.gen(function* () {
     const resolvedRepoPath = yield* trySync2(() => path2.resolve(repoPath));
@@ -4216,19 +4268,27 @@ function exportBackupEffect({
     }
     yield* ensureBackupGitattributesEffect(resolvedRepoPath);
     yield* ensureBackupReadmeEffect(resolvedRepoPath);
+    const maxBytes = yield* trySync2(
+      () => normalizeMaxBackupShardBytes(maxShardBytes)
+    );
     const shards = yield* trySync2(() => buildShards(database));
     const shardEntries = yield* trySync2(
       () => [...shards.entries()].sort(
         ([left], [right]) => left.localeCompare(right)
       )
     );
+    const shardParts = yield* trySync2(
+      () => shardEntries.flatMap(
+        ([relativePath, rows]) => splitJsonlShard(relativePath, rows, maxBytes)
+      )
+    );
     const expectedPaths = yield* trySync2(
-      () => new Set(shardEntries.map(([relativePath]) => relativePath))
+      () => new Set(shardParts.map(({ relativePath }) => relativePath))
     );
     const files = yield* Effect5.forEach(
-      shardEntries,
-      ([relativePath, rows]) => writeJsonlFileEffect(resolvedRepoPath, relativePath, rows),
-      { concurrency: "unbounded" }
+      shardParts,
+      ({ relativePath, rows }) => writeJsonlFileEffect(resolvedRepoPath, relativePath, rows),
+      { concurrency: 1 }
     );
     yield* removeStaleBackupFilesEffect(resolvedRepoPath, expectedPaths);
     const counts = yield* trySync2(() => countBackupFiles(files));
@@ -4416,7 +4476,7 @@ function readBackupImportRowsEffect(resolvedRepoPath, manifest) {
   });
 }
 function rowsForManifestPath(manifest, predicate) {
-  return manifest.files.map((file) => file.path).filter(predicate).sort();
+  return manifest.files.map((file) => file.path).filter((relativePath) => predicate(logicalBackupShardPath(relativePath))).sort();
 }
 function importBackupEffect({
   repoPath,
@@ -4886,7 +4946,7 @@ function validateBackupEffect(repoPath) {
         }
         return { file, errors: fileErrors };
       }),
-      { concurrency: "unbounded" }
+      { concurrency: 1 }
     );
     const files = [];
     for (const result of results) {

package/dist/server/assets/{router-CAvZOdfA.js → router-D8AUR_Gz.js}

@@ -2546,6 +2546,10 @@ var Route$17 = createFileRoute("/api/xurl-rate-limits")({ server: { handlers: {
 })) } } });
 //#endregion
 //#region src/lib/backup-table-codecs.ts
+var BACKUP_SHARD_PART_PATTERN = /\.part-\d{4,}\.jsonl$/u;
+function logicalBackupShardPath(relativePath) {
+	return relativePath.replace(BACKUP_SHARD_PART_PATTERN, ".jsonl");
+}
 function fixedShard(relativePath, countKey) {
 	return {
 		shardPath: () => relativePath,
@@ -3567,7 +3571,8 @@ function adaptLegacyTweetState(schemaVersion, tweets, collections, timelineEdges
 	};
 }
 function backupCodecForPath(relativePath, codecs = BACKUP_TABLE_CODECS) {
-	const matches = codecs.filter((codec) => codec.matchesPath(relativePath));
+	const logicalPath = logicalBackupShardPath(relativePath);
+	const matches = codecs.filter((codec) => codec.matchesPath(logicalPath));
 	if (matches.length !== 1) throw new Error(matches.length === 0 ? `No backup codec owns path: ${relativePath}` : `Multiple backup codecs own path: ${relativePath}`);
 	return matches[0];
 }
@@ -3590,7 +3595,7 @@ function countBackupFiles(files, codecs = BACKUP_TABLE_CODECS) {
 	const counts = {};
 	for (const file of files) {
 		if (!file.path.startsWith("data/")) continue;
-		const key = backupCodecForPath(file.path, codecs).countKey(file.path);
+		const key = backupCodecForPath(file.path, codecs).countKey(logicalBackupShardPath(file.path));
 		counts[key] = (counts[key] ?? 0) + file.rows;
 	}
 	return counts;
@@ -3996,8 +4001,9 @@ function runSubprocessEffect(options) {
 }
 //#endregion
 //#region src/lib/backup.ts
-var BACKUP_SCHEMA_VERSION = 3;
+var BACKUP_SCHEMA_VERSION = 4;
 var MIN_SUPPORTED_BACKUP_SCHEMA_VERSION = 1;
+var DEFAULT_MAX_BACKUP_SHARD_BYTES = 48 * 1024 * 1024;
 var MANIFEST_PATH = "manifest.json";
 var DATA_DIR = "data";
 var GITATTRIBUTES_PATH = ".gitattributes";
@@ -4080,6 +4086,40 @@ function getExportRowSets(db) {
 function buildShards(db) {
 	return buildBackupShardsFromRowSets(getExportRowSets(db));
 }
+function normalizeMaxBackupShardBytes(value) {
+	const maxBytes = value ?? DEFAULT_MAX_BACKUP_SHARD_BYTES;
+	if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0) throw new Error("Backup shard byte limit must be a positive integer");
+	return maxBytes;
+}
+function partPath(relativePath, partNumber) {
+	if (!relativePath.endsWith(".jsonl")) throw new Error(`Backup shard path must end in .jsonl: ${relativePath}`);
+	return relativePath.replace(/\.jsonl$/u, `.part-${String(partNumber).padStart(4, "0")}.jsonl`);
+}
+function splitJsonlShard(relativePath, rows, maxBytes) {
+	const parts = [];
+	let currentRows = [];
+	let currentBytes = 0;
+	for (const [index, row] of rows.entries()) {
+		const rowBytes = Buffer.byteLength(jsonlStringify(row)) + 1;
+		if (rowBytes > maxBytes) throw new Error(`Backup row exceeds shard byte limit: ${relativePath}:${index + 1} is ${rowBytes} bytes (limit ${maxBytes})`);
+		if (currentRows.length > 0 && currentBytes + rowBytes > maxBytes) {
+			parts.push(currentRows);
+			currentRows = [];
+			currentBytes = 0;
+		}
+		currentRows.push(row);
+		currentBytes += rowBytes;
+	}
+	if (currentRows.length > 0) parts.push(currentRows);
+	if (parts.length <= 1) return [{
+		relativePath,
+		rows: parts[0] ?? []
+	}];
+	return parts.map((partRows, index) => ({
+		relativePath: partPath(relativePath, index + 1),
+		rows: partRows
+	}));
+}
 function writeJsonlFileEffect(repoPath, relativePath, rows) {
 	return Effect.gen(function* () {
 		const fullPath = yield* trySync$10(() => resolveBackupFilePath(repoPath, relativePath));
@@ -4165,6 +4205,7 @@ data/follow_events.jsonl
 \`\`\`
 
 Tweets are sharded by creation year. Collection-only tweets whose creation date is unknown live in \`data/tweets/unknown.jsonl\`. Timeline edges keep account-scoped home/mention membership separate from canonical tweet content. DMs are sharded by year and keep \`conversation_id\` in each row.
+Logical shards larger than 48 MiB are split into deterministic \`.part-0001.jsonl\` files so ordinary Git hosting remains usable without Git LFS.
 The links shard stores expanded short URLs and their source tweet/DM occurrences so linked-tweet search can be rebuilt without re-expanding every \`t.co\` URL.
 
 Never commit live tokens, browser cookies, raw SQLite WAL/SHM sidecars, or temporary cache files here.
@@ -4398,7 +4439,7 @@ function pullBackupGitRepoEffect(repoPath) {
 		]).pipe(Effect.as(true), Effect.catchAll(() => Effect.succeed(false)))));
 	});
 }
-function exportBackupEffect({ repoPath, db, commit = false, push = false, message = "archive: update birdclaw backup", validate = true }) {
+function exportBackupEffect({ repoPath, db, commit = false, push = false, message = "archive: update birdclaw backup", validate = true, maxShardBytes }) {
 	return Effect.gen(function* () {
 		const resolvedRepoPath = yield* trySync$10(() => path.resolve(repoPath));
 		const database = db ?? (yield* trySync$10(() => getNativeDb({ seedDemoData: false })));
@@ -4407,10 +4448,12 @@ function exportBackupEffect({ repoPath, db, commit = false, push = false, messag
 		if (!repoStat.isDirectory() || repoStat.isSymbolicLink()) return yield* Effect.fail(/* @__PURE__ */ new Error("Backup repository path must be a real directory"));
 		yield* ensureBackupGitattributesEffect(resolvedRepoPath);
 		yield* ensureBackupReadmeEffect(resolvedRepoPath);
+		const maxBytes = yield* trySync$10(() => normalizeMaxBackupShardBytes(maxShardBytes));
 		const shards = yield* trySync$10(() => buildShards(database));
 		const shardEntries = yield* trySync$10(() => [...shards.entries()].sort(([left], [right]) => left.localeCompare(right)));
-		const expectedPaths = yield* trySync$10(() => new Set(shardEntries.map(([relativePath]) => relativePath)));
-		const files = yield* Effect.forEach(shardEntries, ([relativePath, rows]) => writeJsonlFileEffect(resolvedRepoPath, relativePath, rows), { concurrency: "unbounded" });
+		const shardParts = yield* trySync$10(() => shardEntries.flatMap(([relativePath, rows]) => splitJsonlShard(relativePath, rows, maxBytes)));
+		const expectedPaths = yield* trySync$10(() => new Set(shardParts.map(({ relativePath }) => relativePath)));
+		const files = yield* Effect.forEach(shardParts, ({ relativePath, rows }) => writeJsonlFileEffect(resolvedRepoPath, relativePath, rows), { concurrency: 1 });
 		yield* removeStaleBackupFilesEffect(resolvedRepoPath, expectedPaths);
 		const counts = yield* trySync$10(() => countBackupFiles(files));
 		const backupHash = yield* trySync$10(() => computeBackupHash(files));
@@ -4526,7 +4569,7 @@ function readBackupImportRowsEffect(resolvedRepoPath, manifest) {
 	});
 }
 function rowsForManifestPath(manifest, predicate) {
-	return manifest.files.map((file) => file.path).filter(predicate).sort();
+	return manifest.files.map((file) => file.path).filter((relativePath) => predicate(logicalBackupShardPath(relativePath))).sort();
 }
 function importBackupEffect({ repoPath, db: providedDb, validate = true, mode = "merge" }) {
 	return Effect.gen(function* () {
@@ -4901,7 +4944,7 @@ function validateBackupEffect(repoPath) {
 				file,
 				errors: fileErrors
 			};
-		}), { concurrency: "unbounded" });
+		}), { concurrency: 1 });
 		const files = [];
 		for (const result of results) {
 			errors.push(...result.errors);

package/dist/server/server.js

@@ -5130,7 +5130,7 @@ var getBaseManifest = getProdBaseManifest;
 var createEarlyHintsForRequest = createEarlyHintsCollector;
 async function loadEntries() {
 	const [routerEntry, startEntry, pluginAdapters] = await Promise.all([
-		import("./assets/router-CAvZOdfA.js"),
+		import("./assets/router-D8AUR_Gz.js"),
 		import("./assets/start-FaDOTmto.js"),
 		import("./assets/empty-plugin-adapters-DzrEkdBF.js")
 	]);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "birdclaw",
-	"version": "0.9.1",
+	"version": "0.9.2",
 	"description": "Local Twitter memory in SQLite for archives, DMs, likes, bookmarks, and moderation",
 	"homepage": "https://github.com/steipete/birdclaw#readme",
 	"license": "MIT",