@milaboratories/pl-drivers 1.5.63 → 1.5.65

package/src/drivers/download_blob/sparse_cache/create_sparse_file_script.js

@@ -0,0 +1,123 @@
+/** Test script for creating sparse files on any OS. It also checks if they was created on Windows.
+ * Usage: node script.js <dir where to create a file>
+ * Works on any node >= 18.0.0 */
+
+const { exec } = require('node:child_process');
+const { promisify } = require('node:util');
+const path = require('node:path');
+const fs = require('node:fs/promises');
+
+const testDir = path.resolve(process.argv[2]);
+const baseFileName = 'sparse-test-file.dat';
+const logicalFileSize = 50 * 1024 * 1024; // 50 MB
+
+async function ensureDir(dirPath) {
+  try {
+    await fs.access(dirPath);
+  } catch {
+    await fs.mkdir(dirPath, { recursive: true });
+  }
+}
+
+async function cleanupFile(filePath) {
+  try {
+    await fs.access(filePath);
+    await fs.unlink(filePath);
+  } catch {
+    // File doesn't exist or already removed, ignore
+  }
+}
+
+const asyncExec = promisify(exec);
+
+async function runCommand(commandAndArgs, options) {
+  try {
+    const { stdout, stderr } = await asyncExec(commandAndArgs.join(' '), options);
+
+    return { stdout: stdout?.toString(), stderr: stderr?.toString() };
+  } catch (error) {
+    // execFile promisified version throws an error that includes stdout and stderr
+    console.error(
+      `Command "${commandAndArgs.join(' ')}" failed: ${error.message}`
+    );
+    console.error(`STDOUT: ${error.stdout}`);
+    console.error(`STDERR: ${error.stderr}`);
+    throw error; // Re-throw to be caught by the calling function
+  }
+}
+
+async function createSparseFile(
+  platform,
+  currentFilePath,
+  size
+) {
+  try {
+    await ensureDir(path.dirname(currentFilePath));
+    await cleanupFile(currentFilePath); // Clean up if file exists
+
+    let fileHandle;
+
+    fileHandle = await fs.open(currentFilePath, 'w');
+    await fileHandle.close(); // Close before calling fsutil
+    fileHandle = undefined; // Reset handle
+
+    if (platform === 'win32') {
+      console.log(
+        `Windows: File "${currentFilePath}" created/truncated to ${size} bytes.`
+      );
+      await runCommand(['fsutil', 'sparse', 'setflag', `"${currentFilePath}"`], { stdio: 'pipe' });
+      console.log(`Windows: File "${currentFilePath}" marked as sparse.`);
+    }
+
+    // Write data with gaps to demonstrate sparse nature
+    fileHandle = await fs.open(currentFilePath, 'r+');
+    const initialData = Buffer.from(`BEGIN_SPARSE_${platform.toUpperCase()}_`);
+    await fileHandle.write(initialData, 0, initialData.length, 0);
+
+    const distantData = Buffer.from(`_END_SPARSE_${platform.toUpperCase()}`);
+    const distantOffset = size - distantData.length - 200;
+    await fileHandle.write(
+      distantData,
+      0,
+      distantData.length,
+      distantOffset
+    );
+
+    await fileHandle.close();
+    console.log(`Initial and distant data written to "${currentFilePath}".`);
+    return true;
+  } catch (error) {
+    console.error(
+      `Error in createSparseFile for ${platform} at "${currentFilePath}": ${error.message}`
+    );
+    return false;
+  }
+}
+
+(async () => {
+  const currentPlatform = process.platform;
+  const platformFilePath = path.join(
+    testDir,
+    `${currentPlatform}-${baseFileName}`
+  );
+
+  const creationSuccess = await createSparseFile(
+    currentPlatform,
+    platformFilePath,
+    logicalFileSize
+  );
+
+  if (creationSuccess) {
+    console.log(`Sparse file created successfully at "${platformFilePath}".`);
+  } else {
+    console.error(`Failed to create sparse file at "${platformFilePath}".`);
+  }
+
+  if (currentPlatform === 'win32') {
+    const queryFlag = await runCommand(['fsutil', 'sparse', 'queryflag', `"${platformFilePath}"`], { stdio: 'pipe' });
+    console.log(`Windows: File "${platformFilePath}" is sparse: ${queryFlag.stdout}, error: ${queryFlag.stderr}`);
+
+    const queryRange = await runCommand(['fsutil', 'sparse', 'queryrange', `"${platformFilePath}"`], { stdio: 'pipe' });
+    console.log(`Windows: Query range for "${platformFilePath}": ${queryRange.stdout}, error: ${queryRange.stderr}`);
+  }
+})();

package/src/drivers/download_blob/sparse_cache/file.ts

@@ -0,0 +1,49 @@
+import { fileExists, MiLogger, spawnAsync } from "@milaboratories/ts-helpers";
+import * as fs from 'node:fs/promises';
+
+/** Creates a sparse file for all systems
+ * Table of what supports sparse files:
+ * https://en.wikipedia.org/wiki/Comparison_of_file_systems#Allocation_and_layout_policies */
+export async function createSparseFile(
+  logger: MiLogger,
+  path: string,
+  platform: NodeJS.Platform,
+) {
+  try {
+    const ensureCreated = await fs.open(path, 'w');
+    await ensureCreated.close();
+
+    await ensureSparseOnWindows(path, platform);
+  } catch (error: unknown) {
+    logger.error(`Error creating file ${path} on platform ${platform}: ${error}`);
+  }
+}
+
+/** Sets a sparse flag on Windows.
+ * We could check the file is sparse by running:
+ * `fsutil sparse queryflag <path>`
+ * and
+ * `fsutil sparse queryrange <path>`
+ */
+async function ensureSparseOnWindows(path: string, platform: NodeJS.Platform) {
+  if (platform === 'win32') {
+    await spawnAsync('fsutil', ['sparse', 'setflag', `"${path}"`], { stdio: 'pipe' });
+  }
+}
+
+/** Ensures the file is created and writes to it. */
+export async function writeToSparseFile(
+  logger: MiLogger,
+  platform: NodeJS.Platform,
+  path: string,
+  data: Uint8Array,
+  from: number
+) {
+  if (!(await fileExists(path))) {
+    await createSparseFile(logger, path, platform);
+  }
+
+  const fileHandle = await fs.open(path, 'r+');
+  await fileHandle.write(data, 0, data.length, from);
+  await fileHandle.close();
+}

package/src/drivers/download_blob/sparse_cache/ranges.test.ts

@@ -0,0 +1,115 @@
+import { Ranges, normalizeRanges } from "./ranges";
+import { describe, it, expect } from 'vitest';
+
+describe('normalizeRanges', () => {
+  const cases: { name: string, input: Ranges, expected: Ranges }[] = [
+    {
+      name: 'empty ranges',
+      input: { ranges: [] },
+      expected: { ranges: [] },
+    },
+    {
+      name: 'single range',
+      input: { ranges: [{ from: 0, to: 10 }] },
+      expected: { ranges: [{ from: 0, to: 10 }] },
+    },
+    {
+      name: 'two unsorted non-overlapping ranges',
+      input: {
+        ranges: [
+          { from: 20, to: 30 },
+          { from: 0, to: 10 },
+        ]
+      },
+      expected: {
+        ranges: [
+          { from: 0, to: 10 },
+          { from: 20, to: 30 },
+        ]
+      },
+    },
+    {
+      name: 'two overlapping ranges',
+      input: {
+        ranges: [
+          { from: 0, to: 10 },
+          { from: 5, to: 15 },
+        ]
+      },
+      expected: { ranges: [{ from: 0, to: 15 }] },
+    },
+    {
+      name: 'two adjacent ranges',
+      input: {
+        ranges: [
+          { from: 0, to: 10 },
+          { from: 10, to: 20 },
+        ]
+      },
+      expected: { ranges: [{ from: 0, to: 20 }] },
+    },
+    {
+      name: 'multiple overlapping ranges',
+      input: {
+        ranges: [
+          { from: 0, to: 10 },
+          { from: 5, to: 15 },
+          { from: 12, to: 20 },
+        ]
+      },
+      expected: { ranges: [{ from: 0, to: 20 }] },
+    },
+    {
+      name: 'inner range',
+      input: {
+        ranges: [
+          { from: 0, to: 20 },
+          { from: 5, to: 15 },
+        ]
+      },
+      expected: { ranges: [{ from: 0, to: 20 }] },
+    },
+    {
+      name: 'inner range with outer range',
+      input: {
+        ranges: [
+          { from: 5, to: 15 },
+          { from: 20, to: 30 },
+          { from: 0, to: 20 },
+        ]
+      },
+      expected: { ranges: [{ from: 0, to: 30 }] },
+    },
+    {
+      name: 'more than 1 range in expected',
+      input: {
+        ranges: [
+          { from: 25, to: 30},
+          { from: 20, to: 25 },
+
+          { from: 0, to: 8 },
+          { from: 2, to: 10 },
+          { from: 1, to: 9 },
+
+          { from: 40, to: 50 },
+          { from: 45, to: 47 }
+        ]
+      },
+      expected: { ranges: [
+        { from: 0, to: 10 },
+        { from: 20, to: 30 },
+        { from: 40, to: 50 },
+      ] },
+    }
+  ];
+
+  for (const tc of cases) {
+    it(tc.name, () => {
+      normalizeRanges(tc.input);
+
+      expect(tc.input).toEqual(tc.expected);
+    });
+  }
+});
+
+

package/src/drivers/download_blob/sparse_cache/ranges.ts

@@ -0,0 +1,93 @@
+import { z } from "zod";
+import * as fs from 'node:fs/promises';
+import { RangeBytes } from "@milaboratories/pl-model-common";
+import { createPathAtomically, MiLogger } from "@milaboratories/ts-helpers";
+import { CorruptedRangesError } from "./cache";
+
+/** The content of the ranges file: ranges of bytes.
+ * The ranges should be normalized: sorted and no overlaps.
+ * For that, use `normalizeRanges` function. */
+const Ranges = z.object({
+  ranges: z.array(RangeBytes),
+});
+
+export type Ranges = z.infer<typeof Ranges>;
+
+export const rangesFilePostfix = '.ranges.json';
+
+export function rangesFileName(fPath: string): string {
+  return fPath + rangesFilePostfix;
+}
+
+export async function readRangesFile(logger: MiLogger, path: string): Promise<Ranges> {
+  let ranges: Ranges = { ranges: [] };
+  try {
+    const file = await fs.readFile(path, 'utf8');
+    ranges = Ranges.parse(JSON.parse(file));
+  } catch (e: unknown) {
+
+    if (e instanceof SyntaxError || e instanceof z.ZodError) {
+      const msg = `readRangesFile: the file ${path} was corrupted: ${e}`;
+      logger.error(msg);
+      throw new CorruptedRangesError(msg);
+    }
+
+    if (!(e instanceof Error && 'code' in e && e.code === 'ENOENT')) {
+      throw e;
+    }
+
+    // If the file does not exist, assume the ranges are empty.
+  }
+
+  normalizeRanges(ranges);
+
+  return ranges;
+}
+
+/** Writes to a temporal file and then renames it atomically. */
+export async function writeRangesFile(logger: MiLogger, path: string, ranges: Ranges) {
+  await createPathAtomically(logger, path, async (tempPath: string) => {
+    await fs.writeFile(tempPath, JSON.stringify(ranges, null, 2), { flag: 'wx' });
+  });
+}
+
+/** Sorts and merges overlapping ranges. */
+export function normalizeRanges(s: Ranges) {
+  s.ranges.sort((a, b) => a.from - b.from);
+
+  for (let i = 0; i < s.ranges.length - 1; i++) {
+    if (s.ranges[i].to >= s.ranges[i + 1].from) {
+      mergeRanges(s, i);
+      i--;
+    }
+  }
+}
+
+function mergeRanges(s: Ranges, i: number) {
+  const from = Math.min(s.ranges[i].from, s.ranges[i + 1].from);
+  const to = Math.max(s.ranges[i].to, s.ranges[i + 1].to);
+
+  s.ranges.splice(i, 2, { from, to });
+}
+
+export function rangesSize(s: Ranges) {
+  return s.ranges.reduce((acc, range) => acc + range.to - range.from, 0);
+}
+
+export function doesRangeExist(allRanges: Ranges, range: RangeBytes): boolean {
+  for (const r of allRanges.ranges) {
+    if (r.from <= range.from && range.to <= r.to) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+export function addRange(s: Ranges, range: RangeBytes) {
+  s.ranges.push(range);
+  normalizeRanges(s);
+
+  return s;
+}
+

package/src/drivers/download_url.ts

@@ -58,7 +58,7 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
     httpClient: Dispatcher,
     private readonly saveDir: string,
     private readonly opts: DownloadUrlDriverOps = {
-      cacheSoftSizeBytes: 50 * 1024 * 1024,
+      cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB
       withGunzip: true,
       nConcurrentDownloads: 50,
     },

package/src/drivers/helpers/download_remote_handle.ts

@@ -3,19 +3,20 @@
 
 import type { Signer } from '@milaboratories/ts-helpers';
 import type { OnDemandBlobResourceSnapshot } from '../types';
-import type { RemoteBlobHandle, RangeBytes } from '@milaboratories/pl-model-common';
-import { bigintToResourceId, ResourceId, ResourceType } from '@milaboratories/pl-client';
+import type { RemoteBlobHandle } from '@milaboratories/pl-model-common';
+import { bigintToResourceId } from '@milaboratories/pl-client';
 import { ResourceInfo } from '@milaboratories/pl-tree';
+import { getSize } from '../types';
 
-// https://regex101.com/r/Q4YdTa/4
+// https://regex101.com/r/Q4YdTa/5
 const remoteHandleRegex
-  = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?))#(?<signature>.*)$/;
+  = /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.+)\/(?<resourceVersion>.+?)\/(?<resourceId>\d+?)\/(?<size>\d+?))#(?<signature>.*)$/;
 
 export function newRemoteHandle(
   rInfo: OnDemandBlobResourceSnapshot,
   signer: Signer,
 ): RemoteBlobHandle {
-  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}`;
+  let content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}/${getSize(rInfo)}`;
 
   return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;
 }
@@ -24,18 +25,24 @@ export function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {
   return Boolean(handle.match(remoteHandleRegex));
 }
 
-export function parseRemoteHandle(handle: RemoteBlobHandle, signer: Signer): ResourceInfo {
+export function parseRemoteHandle(handle: RemoteBlobHandle, signer: Signer): {
+  info: ResourceInfo;
+  size: number;
+ } {
   const parsed = handle.match(remoteHandleRegex);
   if (parsed === null) {
     throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);
   }
 
-  const { content, resourceType, resourceVersion, resourceId, signature } = parsed.groups!;
+  const { content, resourceType, resourceVersion, resourceId, size, signature } = parsed.groups!;
 
   signer.verify(content, signature, `Signature verification failed for ${handle}`);
 
   return {
-    id: bigintToResourceId(BigInt(resourceId)),
-    type: { name: resourceType, version: resourceVersion },
+    info:{
+      id: bigintToResourceId(BigInt(resourceId)),
+      type: { name: resourceType, version: resourceVersion },
+    },
+    size: Number(size),
   };
 }

package/src/drivers/logs.test.ts

@@ -25,6 +25,12 @@ import { DownloadDriver } from './download_blob/download_blob';
 import { LogsDriver } from './logs';
 import { LogsStreamDriver } from './logs_stream';
 
+const downloadDriverOps = {
+  cacheSoftSizeBytes: 700 * 1024,
+  nConcurrentDownloads: 10,
+  rangesCacheMaxSizeBytes: 1024,
+};
+
 test('should get all logs', async () => {
   await TestHelpers.withTempRoot(async (client) => {
     const logger = new ConsoleLoggerAdapter();
@@ -35,13 +41,15 @@ test('should get all logs', async () => {
     });
     const logsStream = new LogsStreamDriver(logger, createLogsClient(client, logger));
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-1-'));
-    const download = new DownloadDriver(
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-ranges'));
+    const download = await DownloadDriver.init(
       logger,
       createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
+      rangesCacheDir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
+      downloadDriverOps,
     );
     const logs = new LogsDriver(logger, logsStream, download);
 
@@ -80,13 +88,15 @@ test('should get last line with a prefix', async () => {
     });
     const logsStream = new LogsStreamDriver(logger, createLogsClient(client, logger));
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-2-'));
-    const download = new DownloadDriver(
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-ranges'));
+    const download = await DownloadDriver.init(
       logger,
       createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
+      rangesCacheDir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
+      downloadDriverOps,
     );
     const logs = new LogsDriver(logger, logsStream, download);
 
@@ -133,13 +143,15 @@ test('should get log smart object and get log lines from that', async () => {
     });
     const logsStream = new LogsStreamDriver(logger, createLogsClient(client, logger));
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-3-'));
-    const download = new DownloadDriver(
+    const rangesCacheDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-ranges'));
+    const download = await DownloadDriver.init(
       logger,
       createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
+      rangesCacheDir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
+      downloadDriverOps,
     );
     const logs = new LogsDriver(logger, logsStream, download);