@lamplitisles/dsh-nanocodex 0.1.0-beta.0

package/vendor/nanocodex-tools/tools/dataset.d.mts

@@ -0,0 +1,9 @@
+import type { NamedTool } from "./types.mjs";
+
+export type DatasetOptions = Readonly<{
+  /** Fetch implementation used for dataset metadata, ranges, and streams. */
+  fetch?: typeof globalThis.fetch | undefined;
+}>;
+
+/** A lazy, session-scoped inspector for public Parquet, JSONL, and Hugging Face datasets. */
+export function dataset(options?: DatasetOptions): NamedTool;

package/vendor/nanocodex-tools/tools/dataset.mjs

@@ -0,0 +1,47 @@
+import { namedTool } from "./namedTool.mjs";
+import { DATASET_DESCRIPTION, datasetOutputSchema, datasetParameters } from "./datasetContract.mjs";
+
+/** A session-scoped, lazy browser dataset inspector. */
+export function dataset(options = {}) {
+  if (!options || typeof options !== "object" || Array.isArray(options)) {
+    throw new TypeError("dataset tool options must be an object");
+  }
+  if (options.fetch !== undefined && typeof options.fetch !== "function") {
+    throw new TypeError("dataset tool fetch must be a function");
+  }
+  const engineOptions = Object.freeze({ fetch: options.fetch });
+  let loaded;
+  let resolved;
+  let disposed = false;
+  const releasedSessions = new Set();
+  const tool = namedTool("dataset", {
+    description: DATASET_DESCRIPTION,
+    parameters: datasetParameters,
+    outputSchema: datasetOutputSchema,
+    handler(input, context) {
+      if (disposed) throw new Error("dataset tool is disposed");
+      loaded ??= import("./datasetEngine.mjs")
+        .then(({ createDatasetTool }) => {
+          resolved = createDatasetTool(engineOptions);
+          for (const sessionId of releasedSessions) resolved.releaseSession(sessionId);
+          releasedSessions.clear();
+          if (disposed) resolved.dispose();
+          return resolved;
+        });
+      return loaded.then((tool) => {
+        if (disposed) throw new Error("dataset tool is disposed");
+        return tool.handler(input, context);
+      });
+    },
+    releaseSession(sessionId) {
+      if (resolved) resolved.releaseSession(sessionId);
+      else if (loaded) releasedSessions.add(sessionId);
+    },
+    dispose() {
+      disposed = true;
+      releasedSessions.clear();
+      resolved?.dispose();
+    },
+  });
+  return tool;
+}

package/vendor/nanocodex-tools/tools/datasetContract.mjs

@@ -0,0 +1,54 @@
+export const DATASET_DESCRIPTION = `Inspect public Parquet, uncompressed JSONL, or Hugging Face datasets in the
+browser. Open a URL source or {kind:"huggingface",dataset:"owner/name",config?,split?}; open returns
+schema and a JSONL preview. Query dataset_id with columns, filters, offset, and limit. Continue a
+partial result by passing nextCursor as cursor; it retains projection and filters and resumes physical
+Parquet or JSONL scans. Never infer absence
+when complete is false. Handles assume immutable source URLs; close and reopen to refresh changed data.`;
+
+export const MAX_QUERY_BYTES = 128 * 1024 * 1024;
+
+const sourceSchema = {
+  type: "object",
+  properties: {
+    kind: { type: "string", enum: ["url", "huggingface"] },
+    url: { type: "string" },
+    format: { type: "string", enum: ["parquet", "jsonl"] },
+    dataset: { type: "string" },
+    config: { type: "string" },
+    split: { type: "string" },
+  },
+  required: ["kind"],
+  additionalProperties: false,
+};
+
+export const datasetParameters = {
+  type: "object",
+  properties: {
+    operation: { type: "string", enum: ["open", "query", "close"] },
+    source: sourceSchema,
+    dataset_id: { type: "string" },
+    cursor: { type: "string" },
+    columns: { type: "array", maxItems: 64, items: { type: "string" } },
+    filters: {
+      type: "array",
+      maxItems: 8,
+      items: {
+        type: "object",
+        properties: {
+          column: { type: "string" },
+          op: { type: "string", enum: ["eq", "ne", "gt", "gte", "lt", "lte", "in", "contains"] },
+          value: {},
+        },
+        required: ["column", "op", "value"],
+        additionalProperties: false,
+      },
+    },
+    offset: { type: "integer", minimum: 0 },
+    limit: { type: "integer", minimum: 1 },
+    max_bytes: { type: "integer", minimum: 1024, maximum: MAX_QUERY_BYTES },
+  },
+  required: ["operation"],
+  additionalProperties: false,
+};
+
+export const datasetOutputSchema = { type: "object" };