@lossless.org/objectstorage 11.0.0 → 11.2.0

package/third_party_licenses/toml_parser-1.1.5-license.txt

@@ -0,0 +1,19 @@
+Copyright (c) Individual contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/third_party_licenses/winnow-1.0.4-license.txt

@@ -0,0 +1,18 @@
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@lossless.org/objectstorage',
-  version: '11.0.0',
+  version: '11.2.0',
   description: 'A Node.js TypeScript package to create a local S3-compatible storage server using mapped local directories for development and testing purposes.'
 }

package/ts/classes.dashboard.ts

@@ -0,0 +1,280 @@
+import * as plugins from './dashboard.plugins.js';
+import { files as bundledFiles } from './dashboard.bundled.js';
+import type {
+  IEvent_ObjectStorageDashboardSample,
+  IObjectStorageClusterSnapshot,
+  IObjectStorageDashboardSample,
+  IReq_ObjectStorageDashboardGetSample,
+} from './dashboard.interfaces.js';
+
+/** How often the shared sampler asks the engine for a snapshot. */
+export const DASHBOARD_SAMPLE_INTERVAL_MS = 2_000;
+/** Dashboard sockets admitted at once; the next one is refused. */
+export const DASHBOARD_MAXIMUM_CONNECTIONS = 16;
+/** Largest encoded sample pushed; TypedSocket refuses events above 256 KiB. */
+const MAXIMUM_SAMPLE_BYTES = 200 * 1024;
+const DASHBOARD_TAG = 'objectstorage-dashboard';
+const SAMPLE_EVENT: IEvent_ObjectStorageDashboardSample['name'] = 'objectStorageDashboardSample';
+
+const contentTypes: Record<string, string> = {
+  '.html': 'text/html; charset=utf-8',
+  '.js': 'text/javascript; charset=utf-8',
+  '.css': 'text/css; charset=utf-8',
+  '.json': 'application/json; charset=utf-8',
+  '.txt': 'text/plain; charset=utf-8',
+  '.svg': 'image/svg+xml',
+  '.woff2': 'font/woff2',
+};
+
+export interface IObjectStorageDashboardOptions {
+  address: string;
+  port: number;
+  /** Reads one engine snapshot; called by the shared sampler only. */
+  readSnapshot: () => Promise<IObjectStorageClusterSnapshot>;
+}
+
+/**
+ * The read-only dashboard: the bundled page plus one TypedSocket on its own
+ * port. It reads the engine only through `readSnapshot` (one IPC command),
+ * and only while at least one dashboard is connected: one shared sampler
+ * every two seconds, one sample in flight, missed ticks skipped. Its router
+ * holds one read handler; nothing it serves can change the engine.
+ */
+export class ObjectStorageDashboard {
+  private readonly options: IObjectStorageDashboardOptions;
+  private readonly router = new plugins.typedrequest.TypedRouter();
+  private readonly typedSocket: plugins.typedsocket.TypedSocket;
+  private readonly smartServe: plugins.smartserve.SmartServe;
+  private boundPort = 0;
+  private timer: ReturnType<typeof setInterval> | undefined;
+  private inFlight: Promise<IObjectStorageDashboardSample> | undefined;
+  private latestSample: IObjectStorageDashboardSample | undefined;
+  private publishFailureLogged = false;
+  private stopped = false;
+
+  constructor(optionsArg: IObjectStorageDashboardOptions) {
+    this.options = optionsArg;
+    this.router.addTypedHandler(
+      new plugins.typedrequest.TypedHandler<IReq_ObjectStorageDashboardGetSample>(
+        'objectStorageDashboardGetSample',
+        async () => this.latestSample ?? (await this.takeSample()),
+      ),
+    );
+    this.typedSocket = plugins.typedsocket.TypedSocket.createServer(this.router, {
+      onServerConnectionReady: (connection) => {
+        this.typedSocket.setServerTag(connection, DASHBOARD_TAG);
+        this.startSampler();
+        return undefined;
+      },
+    });
+    this.smartServe = new plugins.smartserve.SmartServe({
+      port: optionsArg.port,
+      hostname: optionsArg.address,
+      websocket: {
+        typedRouter: this.typedSocket.getServerRoutingSurface(this.router),
+        transportOwner: this.typedSocket.webSocketTransportOwner,
+        admit: () => {
+          if (this.connectionCount() >= DASHBOARD_MAXIMUM_CONNECTIONS) {
+            return new Response(
+              `The dashboard admits ${DASHBOARD_MAXIMUM_CONNECTIONS} connections at once.`,
+              { status: 503, headers: { 'Retry-After': '10' } },
+            );
+          }
+          return true;
+        },
+        onConnectionOpen: (peer) => {
+          // Admission and registration are not one step: upgrades admitted
+          // together can pass the check before any of them registers.
+          if (this.connectionCount() > DASHBOARD_MAXIMUM_CONNECTIONS) {
+            peer.close(1013, 'Dashboard connection limit reached');
+          }
+        },
+      },
+    });
+    this.typedSocket.attachSmartServe(this.smartServe);
+    this.addStaticRoutes();
+  }
+
+  /** The bound TCP port while running. */
+  public get port(): number {
+    return this.boundPort;
+  }
+
+  public async start(): Promise<void> {
+    const instance = await this.smartServe.start();
+    this.boundPort = instance.port;
+  }
+
+  /** Stops the sampler, waits for a sample in flight, then closes every socket. */
+  public async stop(): Promise<void> {
+    this.stopped = true;
+    this.stopSampler();
+    if (this.inFlight) {
+      await this.inFlight;
+    }
+    await this.typedSocket.stop();
+    await this.smartServe.stop();
+    this.boundPort = 0;
+  }
+
+  private connectionCount(): number {
+    return this.smartServe.getWebSocketConnections().length;
+  }
+
+  private addStaticRoutes(): void {
+    const index = bundledFiles.find((file) => file.path === 'index.html');
+    if (!index) {
+      throw new Error('The dashboard bundle has no index.html; run `tsrepo bundle`.');
+    }
+    const serve = (file: { path: string; contentBase64: string }) => {
+      const body = Buffer.from(file.contentBase64, 'base64');
+      const extension = file.path.slice(file.path.lastIndexOf('.'));
+      const headers = {
+        'Content-Type': contentTypes[extension] ?? 'application/octet-stream',
+        'Cache-Control': 'no-cache',
+        'X-Content-Type-Options': 'nosniff',
+        'Referrer-Policy': 'no-referrer',
+      };
+      return () => new Response(body, { status: 200, headers });
+    };
+    this.smartServe.addRoute('/', 'GET', serve(index));
+    for (const file of bundledFiles) {
+      this.smartServe.addRoute(`/${file.path}`, 'GET', serve(file));
+    }
+  }
+
+  private startSampler(): void {
+    if (this.timer || this.stopped) {
+      return;
+    }
+    this.timer = setInterval(() => this.tick(), DASHBOARD_SAMPLE_INTERVAL_MS);
+    this.tick();
+  }
+
+  private stopSampler(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+  }
+
+  private tick(): void {
+    if (this.stopped) {
+      return;
+    }
+    if (this.connectionCount() === 0) {
+      this.stopSampler();
+      this.latestSample = undefined;
+      return;
+    }
+    if (this.inFlight) {
+      return;
+    }
+    void this.takeSample().then((sample) => this.publish(sample));
+  }
+
+  /**
+   * Pushes one sample to every dashboard. Never throws: it ends the sampler's
+   * promise chain, so a throw here would be an unhandled rejection.
+   */
+  private publish(sampleArg: IObjectStorageDashboardSample): void {
+    if (this.stopped) {
+      return;
+    }
+    let sampleFailure: unknown;
+    try {
+      this.typedSocket.publishToTag<IEvent_ObjectStorageDashboardSample>(
+        DASHBOARD_TAG,
+        SAMPLE_EVENT,
+        sampleArg,
+      );
+      this.publishFailureLogged = false;
+      return;
+    } catch (error) {
+      sampleFailure = error;
+    }
+    // A sample TypedSocket refuses (too large even without peer events)
+    // reaches the dashboards as an error instead of silence.
+    try {
+      this.typedSocket.publishToTag<IEvent_ObjectStorageDashboardSample>(
+        DASHBOARD_TAG,
+        SAMPLE_EVENT,
+        {
+          sampledAt: sampleArg.sampledAt,
+          snapshot: null,
+          error: `The sample could not be pushed: ${describeError(sampleFailure)}`,
+          eventsTruncated: false,
+        },
+      );
+      this.publishFailureLogged = false;
+    } catch (errorPushFailure) {
+      // Neither reached the dashboards, so the process log is the only
+      // channel left: once per run of failures, not every tick.
+      if (!this.publishFailureLogged) {
+        this.publishFailureLogged = true;
+        console.error(
+          `objectstorage dashboard: a sample could not be pushed (${describeError(sampleFailure)}),`
+            + ` nor its error (${describeError(errorPushFailure)}).`,
+        );
+      }
+    }
+  }
+
+  /** Takes one sample, or joins the one in flight. Never rejects. */
+  private takeSample(): Promise<IObjectStorageDashboardSample> {
+    if (this.inFlight) {
+      return this.inFlight;
+    }
+    const sample = this.readSample().finally(() => {
+      this.inFlight = undefined;
+    });
+    this.inFlight = sample;
+    return sample;
+  }
+
+  private async readSample(): Promise<IObjectStorageDashboardSample> {
+    const sampledAt = Date.now();
+    let sample: IObjectStorageDashboardSample;
+    try {
+      const snapshot = await this.options.readSnapshot();
+      sample = fitSample({ sampledAt, snapshot, error: null, eventsTruncated: false });
+    } catch (error) {
+      sample = {
+        sampledAt,
+        snapshot: null,
+        error: describeError(error),
+        eventsTruncated: false,
+      };
+    }
+    this.latestSample = sample;
+    return sample;
+  }
+}
+
+const describeError = (errorArg: unknown): string =>
+  errorArg instanceof Error ? errorArg.message : String(errorArg);
+
+/**
+ * Keeps a sample within the push limit by halving the events of the peer
+ * snapshots until it fits; the local node keeps all of its events.
+ */
+const fitSample = (sampleArg: IObjectStorageDashboardSample): IObjectStorageDashboardSample => {
+  const encodedBytes = (value: IObjectStorageDashboardSample) =>
+    Buffer.byteLength(JSON.stringify(value));
+  const peers = sampleArg.snapshot?.cluster?.peers ?? [];
+  let sample = sampleArg;
+  while (encodedBytes(sample) > MAXIMUM_SAMPLE_BYTES) {
+    const longest = Math.max(0, ...peers.map((peer) => peer.snapshot?.events.length ?? 0));
+    if (longest === 0) {
+      break;
+    }
+    const keep = Math.floor(longest / 2);
+    for (const peer of peers) {
+      if (peer.snapshot && peer.snapshot.events.length > keep) {
+        peer.snapshot.events = peer.snapshot.events.slice(peer.snapshot.events.length - keep);
+      }
+    }
+    sample = { ...sample, eventsTruncated: true };
+  }
+  return sample;
+};