@elaraai/e3-core 0.0.2-beta.4 → 0.0.2-beta.41

package/dist/src/trees.js

@@ -12,17 +12,16 @@
  * tree node. This enables proper encoding/decoding with the correct StructType
  * and supports future tree types (array, variant, etc.).
  *
- * Low-level operations work with hashes directly (by-hash).
- * High-level operations traverse paths from a root (by-path).
+ * Workspace operations use per-dataset ref files (DatasetRef) instead of tree
+ * traversal. This enables concurrent writes without serialization.
+ *
+ * Low-level tree read/write operations remain for computing root hashes and
+ * for package operations.
  */
-import { decodeBeast2, decodeBeast2For, encodeBeast2For, StructType, } from '@elaraai/east';
+import { decodeBeast2, decodeBeast2For, encodeBeast2For, StructType, variant, } from '@elaraai/east';
 import { DataRefType, PackageObjectType, WorkspaceStateType } from '@elaraai/e3-types';
-import { objectRead, objectWrite } from './objects.js';
 import { packageRead } from './packages.js';
-import { WorkspaceNotFoundError, WorkspaceNotDeployedError, isNotFoundError, } from './errors.js';
-import { acquireWorkspaceLock } from './workspaceLock.js';
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import { WorkspaceNotFoundError, WorkspaceNotDeployedError, WorkspaceLockError, } from './errors.js';
 /**
  * Build the EastType for a tree object based on its structure.
  *
@@ -50,32 +49,34 @@ function treeTypeFromStructure(structure) {
 /**
  * Read and decode a tree object from the object store.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param hash - Hash of the tree object
  * @param structure - The structure describing this tree node's shape
  * @returns The decoded tree object (field name -> DataRef)
  * @throws If object not found, structure is not a tree, or decoding fails
  */
-export async function treeRead(repoPath, hash, structure) {
+export async function treeRead(storage, repo, hash, structure) {
     const treeType = treeTypeFromStructure(structure);
-    const data = await objectRead(repoPath, hash);
+    const data = await storage.objects.read(repo, hash);
     const decoder = decodeBeast2For(treeType);
     return decoder(Buffer.from(data));
 }
 /**
  * Encode and write a tree object to the object store.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param fields - Object mapping field names to DataRefs
  * @param structure - The structure describing this tree node's shape
  * @returns Hash of the written tree object
  * @throws If structure is not a tree or encoding fails
  */
-export async function treeWrite(repoPath, fields, structure) {
+export async function treeWrite(storage, repo, fields, structure) {
     const treeType = treeTypeFromStructure(structure);
     const encoder = encodeBeast2For(treeType);
     const data = encoder(fields);
-    return objectWrite(repoPath, data);
+    return storage.objects.write(repo, data);
 }
 /**
  * Read and decode a dataset value from the object store.
@@ -83,177 +84,42 @@ export async function treeWrite(repoPath, fields, structure) {
  * The .beast2 format includes type information in the header, so values
  * can be decoded without knowing the schema in advance.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param hash - Hash of the dataset value
  * @returns The decoded value and its type
  * @throws If object not found or not a valid beast2 object
  */
-export async function datasetRead(repoPath, hash) {
-    const data = await objectRead(repoPath, hash);
+export async function datasetRead(storage, repo, hash) {
+    const data = await storage.objects.read(repo, hash);
     const result = decodeBeast2(Buffer.from(data));
     return { type: result.type, value: result.value };
 }
 /**
  * Encode and write a dataset value to the object store.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param value - The value to encode
  * @param type - The East type for encoding (EastType or EastTypeValue)
  * @returns Hash of the written dataset value
  */
-export async function datasetWrite(repoPath, value, type) {
+export async function datasetWrite(storage, repo, value, type) {
     // encodeBeast2For accepts both EastType and EastTypeValue, but TypeScript
     // overloads don't support union types directly. Cast to EastTypeValue since
     // that's the more general case and the runtime handles both.
     const encoder = encodeBeast2For(type);
     const data = encoder(value);
-    return objectWrite(repoPath, data);
-}
-/**
- * Traverse a tree from root to a path, co-walking structure and data.
- *
- * @param repoPath - Path to .e3 repository
- * @param rootHash - Hash of the root tree object
- * @param rootStructure - Structure of the root tree
- * @param path - Path to traverse
- * @returns The structure and DataRef at the path location
- * @throws If path is invalid or traversal fails
- */
-async function traverse(repoPath, rootHash, rootStructure, path) {
-    let currentStructure = rootStructure;
-    let currentHash = rootHash;
-    for (let i = 0; i < path.length; i++) {
-        const segment = path[i];
-        if (segment.type !== 'field') {
-            throw new Error(`Unsupported path segment type: ${segment.type}`);
-        }
-        const fieldName = segment.value;
-        // Current structure must be a struct tree to descend into
-        if (currentStructure.type !== 'struct') {
-            const pathSoFar = path.slice(0, i).map(s => s.value).join('.');
-            throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
-        }
-        // Read the current tree object
-        const treeObject = await treeRead(repoPath, currentHash, currentStructure);
-        // Look up the child ref
-        const childRef = treeObject[fieldName];
-        if (!childRef) {
-            const pathSoFar = path.slice(0, i).map(s => s.value).join('.');
-            const available = Object.keys(treeObject).join(', ');
-            throw new Error(`Field '${fieldName}' not found at '${pathSoFar}'. Available: ${available}`);
-        }
-        // Look up the child structure
-        const childStructure = currentStructure.value.get(fieldName);
-        if (!childStructure) {
-            throw new Error(`Field '${fieldName}' not found in structure`);
-        }
-        // If this is the last segment, return the result
-        if (i === path.length - 1) {
-            return { structure: childStructure, ref: childRef };
-        }
-        // Otherwise, continue traversing (must be a tree ref)
-        if (childRef.type !== 'tree') {
-            const pathSoFar = path.slice(0, i + 1).map(s => s.value).join('.');
-            throw new Error(`Expected tree ref at '${pathSoFar}', got '${childRef.type}'`);
-        }
-        currentStructure = childStructure;
-        currentHash = childRef.value;
-    }
-    // Empty path - return root
-    return {
-        structure: rootStructure,
-        ref: { type: 'tree', value: rootHash },
-    };
-}
-/**
- * List field names at a tree path within a package's data tree.
- *
- * @param repoPath - Path to .e3 repository
- * @param name - Package name
- * @param version - Package version
- * @param path - Path to the tree node
- * @returns Array of field names at the path
- * @throws If package not found, path invalid, or path points to a dataset
- */
-export async function packageListTree(repoPath, name, version, path) {
-    // Read the package to get root structure and hash
-    const pkg = await packageRead(repoPath, name, version);
-    const rootStructure = pkg.data.structure;
-    const rootHash = pkg.data.value;
-    if (path.length === 0) {
-        // Empty path - list root tree fields
-        if (rootStructure.type !== 'struct') {
-            throw new Error('Root is not a tree');
-        }
-        const treeObject = await treeRead(repoPath, rootHash, rootStructure);
-        return Object.keys(treeObject);
-    }
-    // Traverse to the path
-    const { structure, ref } = await traverse(repoPath, rootHash, rootStructure, path);
-    // Must be a tree structure
-    if (structure.type !== 'struct') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
-    }
-    // Must be a tree ref
-    if (ref.type !== 'tree') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
-    }
-    // Read the tree and return field names
-    const treeObject = await treeRead(repoPath, ref.value, structure);
-    return Object.keys(treeObject);
-}
-/**
- * Read and decode a dataset value at a path within a package's data tree.
- *
- * @param repoPath - Path to .e3 repository
- * @param name - Package name
- * @param version - Package version
- * @param path - Path to the dataset
- * @returns The decoded dataset value
- * @throws If package not found, path invalid, or path points to a tree
- */
-export async function packageGetDataset(repoPath, name, version, path) {
-    // Read the package to get root structure and hash
-    const pkg = await packageRead(repoPath, name, version);
-    const rootStructure = pkg.data.structure;
-    const rootHash = pkg.data.value;
-    if (path.length === 0) {
-        throw new Error('Cannot get dataset at root path - root is always a tree');
-    }
-    // Traverse to the path
-    const { structure, ref } = await traverse(repoPath, rootHash, rootStructure, path);
-    // Must be a value structure
-    if (structure.type !== 'value') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
-    }
-    // Handle different ref types
-    if (ref.type === 'unassigned') {
-        throw new Error(`Dataset at path is unassigned (pending task output)`);
-    }
-    if (ref.type === 'null') {
-        return null;
-    }
-    if (ref.type === 'tree') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
-    }
-    // Read and return the dataset value
-    const result = await datasetRead(repoPath, ref.value);
-    return result.value;
+    return storage.objects.write(repo, data);
 }
 /**
  * Update a dataset at a path within a workspace.
  *
- * This creates new tree objects along the path with structural sharing,
- * then atomically updates the workspace root.
+ * Writes the value to the object store and updates the per-dataset ref file.
+ * Uses shared structure lock to allow concurrent writes.
  *
- * Acquires an exclusive lock on the workspace for the duration of the write
- * to prevent concurrent modifications.
- *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
  * @param treePath - Path to the dataset
  * @param value - The new value to write
@@ -262,375 +128,462 @@ export async function packageGetDataset(repoPath, name, version, path) {
  * @throws {WorkspaceLockError} If workspace is locked by another process
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
-export async function workspaceSetDataset(repoPath, ws, treePath, value, type, options = {}) {
+export async function workspaceSetDataset(storage, repo, ws, treePath, value, type, options = {}) {
     if (treePath.length === 0) {
         throw new Error('Cannot set dataset at root path - root is always a tree');
     }
     // Acquire lock if not provided externally
     const externalLock = options.lock;
-    const lock = externalLock ?? await acquireWorkspaceLock(repoPath, ws);
-    try {
-        await workspaceSetDatasetUnlocked(repoPath, ws, treePath, value, type);
-    }
-    finally {
-        // Only release the lock if we acquired it internally
-        if (!externalLock) {
-            await lock.release();
+    let lock = externalLock ?? null;
+    if (!lock) {
+        lock = await storage.locks.acquire(repo, ws, variant('dataset_write', null), { mode: 'shared' });
+        if (!lock) {
+            const state = await storage.locks.getState(repo, ws);
+            throw new WorkspaceLockError(ws, state ? {
+                acquiredAt: state.acquiredAt.toISOString(),
+                operation: state.operation.type,
+            } : undefined);
         }
     }
-}
-/**
- * Internal: Update a dataset without acquiring a lock.
- * Caller must hold the workspace lock.
- */
-async function workspaceSetDatasetUnlocked(repoPath, ws, treePath, value, type) {
-    const state = await readWorkspaceState(repoPath, ws);
-    // Read the deployed package object to get the structure
-    const pkgData = await objectRead(repoPath, state.packageHash);
-    const decoder = decodeBeast2For(PackageObjectType);
-    const pkgObject = decoder(Buffer.from(pkgData));
-    const rootStructure = pkgObject.data.structure;
-    // Validate that the path leads to a value structure
-    let currentStructure = rootStructure;
-    for (let i = 0; i < treePath.length; i++) {
-        const segment = treePath[i];
-        if (segment.type !== 'field') {
-            throw new Error(`Unsupported path segment type: ${segment.type}`);
+    try {
+        const wsState = await readWorkspaceState(storage, repo, ws);
+        // Read the deployed package object to get the structure
+        const pkgData = await storage.objects.read(repo, wsState.packageHash);
+        const decoder = decodeBeast2For(PackageObjectType);
+        const pkgObject = decoder(Buffer.from(pkgData));
+        const rootStructure = pkgObject.data.structure;
+        // Validate that the path leads to a value structure and check writable
+        let currentStructure = rootStructure;
+        for (let i = 0; i < treePath.length; i++) {
+            const segment = treePath[i];
+            if (segment.type !== 'field') {
+                throw new Error(`Unsupported path segment type: ${segment.type}`);
+            }
+            if (currentStructure.type !== 'struct') {
+                const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
+                throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
+            }
+            const childStructure = currentStructure.value.get(segment.value);
+            if (!childStructure) {
+                const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
+                const available = Array.from(currentStructure.value.keys()).join(', ');
+                throw new Error(`Field '${segment.value}' not found at '${pathSoFar}'. Available: ${available}`);
+            }
+            currentStructure = childStructure;
         }
-        if (currentStructure.type !== 'struct') {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
+        // Final structure must be a value
+        if (currentStructure.type !== 'value') {
+            const pathStr = treePath.map(s => s.value).join('.');
+            throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
         }
-        const childStructure = currentStructure.value.get(segment.value);
-        if (!childStructure) {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            const available = Array.from(currentStructure.value.keys()).join(', ');
-            throw new Error(`Field '${segment.value}' not found at '${pathSoFar}'. Available: ${available}`);
+        // Check writable flag
+        if (!currentStructure.value.writable) {
+            const pathStr = treePath.map(s => s.value).join('.');
+            throw new Error(`Dataset at '${pathStr}' is not writable`);
         }
-        currentStructure = childStructure;
-    }
-    // Final structure must be a value
-    if (currentStructure.type !== 'value') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
+        // Write the new dataset value to object store
+        const newValueHash = await datasetWrite(storage, repo, value, type);
+        // Build ref path from tree path
+        const refPath = treePath.map(s => s.value).join('/');
+        // Write the DatasetRef with empty version vector (will be populated by dataflow)
+        const datasetRef = variant('value', { hash: newValueHash, versions: new Map() });
+        await storage.datasets.write(repo, ws, refPath, datasetRef);
     }
-    // Write the new dataset value
-    const newValueHash = await datasetWrite(repoPath, value, type);
-    // Now rebuild the tree path from leaf to root (structural sharing)
-    // We need to read each tree along the path, modify it, and write a new version
-    // Collect all tree hashes and structures along the path
-    const treeInfos = [];
-    let currentHash = state.rootHash;
-    currentStructure = rootStructure;
-    // Read all trees along the path (except the last segment which is the dataset)
-    for (let i = 0; i < treePath.length - 1; i++) {
-        treeInfos.push({ hash: currentHash, structure: currentStructure });
-        const segment = treePath[i];
-        const treeObject = await treeRead(repoPath, currentHash, currentStructure);
-        const childRef = treeObject[segment.value];
-        if (!childRef || childRef.type !== 'tree') {
-            throw new Error(`Expected tree ref at path segment ${i}`);
+    finally {
+        // Only release the lock if we acquired it internally
+        if (!externalLock) {
+            await lock.release();
         }
-        currentHash = childRef.value;
-        currentStructure = currentStructure.value.get(segment.value);
-    }
-    // Add the final tree that contains the dataset
-    treeInfos.push({ hash: currentHash, structure: currentStructure });
-    // Now rebuild from leaf to root
-    // Start with the new value hash as the new ref
-    let newRef = { type: 'value', value: newValueHash };
-    for (let i = treeInfos.length - 1; i >= 0; i--) {
-        const { hash, structure } = treeInfos[i];
-        const fieldName = treePath[i].value;
-        // Read the current tree
-        const treeObject = await treeRead(repoPath, hash, structure);
-        // Create modified tree with the new ref
-        const newTreeObject = {
-            ...treeObject,
-            [fieldName]: newRef,
-        };
-        // Write the new tree
-        const newTreeHash = await treeWrite(repoPath, newTreeObject, structure);
-        // This becomes the new ref for the parent
-        newRef = { type: 'tree', value: newTreeHash };
-    }
-    // The final newRef is always a tree ref pointing to the new root
-    // (because we start with a value ref and wrap it in tree refs bottom-up)
-    if (newRef.type !== 'tree' || newRef.value === null) {
-        throw new Error('Internal error: expected tree ref after rebuilding path');
     }
-    const newRootHash = newRef.value;
-    // Update workspace state atomically
-    await writeWorkspaceState(repoPath, ws, {
-        ...state,
-        rootHash: newRootHash,
-        rootUpdatedAt: new Date(),
-    });
 }
 // =============================================================================
 // Workspace Helper Functions
 // =============================================================================
-/**
- * Write workspace state to file atomically.
- */
-async function writeWorkspaceState(repoPath, ws, state) {
-    const wsDir = path.join(repoPath, 'workspaces');
-    const stateFile = path.join(wsDir, `${ws}.beast2`);
-    // Ensure workspaces directory exists
-    await fs.mkdir(wsDir, { recursive: true });
-    const encoder = encodeBeast2For(WorkspaceStateType);
-    const data = encoder(state);
-    // Write atomically: write to temp file, then rename
-    const randomSuffix = Math.random().toString(36).slice(2, 10);
-    const tempPath = path.join(wsDir, `.${ws}.${Date.now()}.${randomSuffix}.tmp`);
-    await fs.writeFile(tempPath, data);
-    await fs.rename(tempPath, stateFile);
-}
 /**
  * Read workspace state from file.
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
  * @throws {WorkspaceNotDeployedError} If workspace exists but not deployed
  */
-async function readWorkspaceState(repoPath, ws) {
-    const stateFile = path.join(repoPath, 'workspaces', `${ws}.beast2`);
-    try {
-        const data = await fs.readFile(stateFile);
-        if (data.length === 0) {
-            throw new WorkspaceNotDeployedError(ws);
-        }
-        const decoder = decodeBeast2For(WorkspaceStateType);
-        return decoder(data);
+async function readWorkspaceState(storage, repo, ws) {
+    const data = await storage.refs.workspaceRead(repo, ws);
+    if (data === null) {
+        throw new WorkspaceNotFoundError(ws);
     }
-    catch (err) {
-        if (err instanceof WorkspaceNotDeployedError)
-            throw err;
-        if (isNotFoundError(err)) {
-            throw new WorkspaceNotFoundError(ws);
-        }
-        throw err;
+    if (data.length === 0) {
+        throw new WorkspaceNotDeployedError(ws);
     }
+    const decoder = decodeBeast2For(WorkspaceStateType);
+    return decoder(data);
 }
 /**
- * Get root structure and hash for a workspace.
+ * Get root structure for a workspace.
  * Reads the deployed package object to get the structure.
  */
-async function getWorkspaceRootInfo(repoPath, ws) {
-    const state = await readWorkspaceState(repoPath, ws);
+async function getWorkspaceStructure(storage, repo, ws) {
+    const wsState = await readWorkspaceState(storage, repo, ws);
     // Read the deployed package object using the stored hash
-    const pkgData = await objectRead(repoPath, state.packageHash);
+    const pkgData = await storage.objects.read(repo, wsState.packageHash);
     const decoder = decodeBeast2For(PackageObjectType);
     const pkgObject = decoder(Buffer.from(pkgData));
     return {
-        rootHash: state.rootHash,
         rootStructure: pkgObject.data.structure,
     };
 }
 // =============================================================================
-// Workspace High-level Operations (by path)
+// Workspace High-level Operations (by path) - Using per-dataset refs
 // =============================================================================
 /**
  * List field names at a tree path within a workspace's data tree.
  *
- * @param repoPath - Path to .e3 repository
+ * Uses the structure to determine available fields (no tree traversal needed).
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
- * @param path - Path to the tree node
+ * @param treePath - Path to the tree node
  * @returns Array of field names at the path
  * @throws If workspace not deployed, path invalid, or path points to a dataset
  */
-export async function workspaceListTree(repoPath, ws, treePath) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(repoPath, ws);
-    if (treePath.length === 0) {
-        // Empty path - list root tree fields
-        if (rootStructure.type !== 'struct') {
-            throw new Error('Root is not a tree');
+export async function workspaceListTree(storage, repo, ws, treePath) {
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
+    // Navigate structure to find the target node
+    let currentStructure = rootStructure;
+    for (let i = 0; i < treePath.length; i++) {
+        const segment = treePath[i];
+        if (segment.type !== 'field') {
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        }
+        if (currentStructure.type !== 'struct') {
+            const pathStr = treePath.slice(0, i).map(s => s.value).join('.');
+            throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
+        }
+        const childStructure = currentStructure.value.get(segment.value);
+        if (!childStructure) {
+            throw new Error(`Field '${segment.value}' not found in structure`);
         }
-        const treeObject = await treeRead(repoPath, rootHash, rootStructure);
-        return Object.keys(treeObject);
+        currentStructure = childStructure;
     }
-    // Traverse to the path
-    const { structure, ref } = await traverse(repoPath, rootHash, rootStructure, treePath);
-    // Must be a tree structure
-    if (structure.type !== 'struct') {
+    if (currentStructure.type !== 'struct') {
         const pathStr = treePath.map(s => s.value).join('.');
         throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
     }
-    // Must be a tree ref
-    if (ref.type !== 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
-    }
-    // Read the tree and return field names
-    const treeObject = await treeRead(repoPath, ref.value, structure);
-    return Object.keys(treeObject);
+    return Array.from(currentStructure.value.keys());
 }
 /**
  * Read and decode a dataset value at a path within a workspace's data tree.
  *
- * @param repoPath - Path to .e3 repository
+ * Reads the per-dataset ref file to get the value hash, then decodes the value.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
- * @param path - Path to the dataset
+ * @param treePath - Path to the dataset
  * @returns The decoded dataset value
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
-export async function workspaceGetDataset(repoPath, ws, treePath) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(repoPath, ws);
+export async function workspaceGetDataset(storage, repo, ws, treePath) {
     if (treePath.length === 0) {
         throw new Error('Cannot get dataset at root path - root is always a tree');
     }
-    // Traverse to the path
-    const { structure, ref } = await traverse(repoPath, rootHash, rootStructure, treePath);
-    // Must be a value structure
-    if (structure.type !== 'value') {
+    // Validate path against structure
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
+    let currentStructure = rootStructure;
+    for (let i = 0; i < treePath.length; i++) {
+        const segment = treePath[i];
+        if (segment.type !== 'field')
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        if (currentStructure.type !== 'struct')
+            throw new Error(`Cannot descend into non-struct`);
+        const child = currentStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found in structure`);
+        currentStructure = child;
+    }
+    if (currentStructure.type !== 'value') {
         const pathStr = treePath.map(s => s.value).join('.');
         throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
     }
-    // Handle different ref types
-    if (ref.type === 'unassigned') {
+    // Read the ref file
+    const refPath = treePath.map(s => s.value).join('/');
+    const ref = await storage.datasets.read(repo, ws, refPath);
+    if (!ref || ref.type === 'unassigned') {
         throw new Error(`Dataset at path is unassigned (pending task output)`);
     }
     if (ref.type === 'null') {
         return null;
     }
-    if (ref.type === 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
-    }
     // Read and return the dataset value
-    const result = await datasetRead(repoPath, ref.value);
+    const result = await datasetRead(storage, repo, ref.value.hash);
     return result.value;
 }
 /**
  * Get the hash of a dataset at a path within a workspace's data tree.
  *
- * Unlike workspaceGetDataset which decodes the value, this returns the raw
- * hash reference. Useful for dataflow execution which operates on hashes.
+ * Reads the per-dataset ref file directly. No tree traversal needed.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
  * @param treePath - Path to the dataset
  * @returns Object with ref type and hash (null for unassigned/null refs)
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
-export async function workspaceGetDatasetHash(repoPath, ws, treePath) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(repoPath, ws);
+export async function workspaceGetDatasetHash(storage, repo, ws, treePath) {
     if (treePath.length === 0) {
         throw new Error('Cannot get dataset at root path - root is always a tree');
     }
-    // Traverse to the path
-    const { structure, ref } = await traverse(repoPath, rootHash, rootStructure, treePath);
-    // Must be a value structure
-    if (structure.type !== 'value') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
-    }
-    // Return ref type and hash
-    if (ref.type === 'unassigned' || ref.type === 'null') {
-        return { refType: ref.type, hash: null };
+    // Read the ref file directly using the path
+    const refPath = treePath.map(s => s.value).join('/');
+    const ref = await storage.datasets.read(repo, ws, refPath);
+    if (!ref || ref.type === 'unassigned') {
+        return { refType: 'unassigned', hash: null };
     }
-    if (ref.type === 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
+    if (ref.type === 'null') {
+        return { refType: 'null', hash: null };
     }
-    return { refType: ref.type, hash: ref.value };
+    return { refType: 'value', hash: ref.value.hash };
 }
 /**
  * Set a dataset at a path within a workspace using a pre-computed hash.
  *
- * Unlike workspaceSetDataset which encodes a value, this takes a hash
- * directly. Useful for dataflow execution which already has the output hash.
+ * Writes a DatasetRef file directly. No tree path-copy needed.
  *
  * IMPORTANT: This function does NOT acquire a workspace lock. The caller must
- * hold an exclusive lock on the workspace (via acquireWorkspaceLock) before
- * calling this function. This is typically used by dataflowExecute which
- * holds the lock for the entire execution.
+ * hold a lock on the workspace before calling this function. This
+ * is typically used by dataflowExecute which holds the lock for the entire
+ * execution.
  *
- * @param repoPath - Path to .e3 repository
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
  * @param ws - Workspace name
  * @param treePath - Path to the dataset
  * @param valueHash - Hash of the dataset value already in the object store
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
-export async function workspaceSetDatasetByHash(repoPath, ws, treePath, valueHash) {
+export async function workspaceSetDatasetByHash(storage, repo, ws, treePath, valueHash, versions) {
     if (treePath.length === 0) {
         throw new Error('Cannot set dataset at root path - root is always a tree');
     }
-    const state = await readWorkspaceState(repoPath, ws);
-    // Read the deployed package object to get the structure
-    const pkgData = await objectRead(repoPath, state.packageHash);
-    const decoder = decodeBeast2For(PackageObjectType);
-    const pkgObject = decoder(Buffer.from(pkgData));
-    const rootStructure = pkgObject.data.structure;
-    // Validate that the path leads to a value structure
+    // Write the DatasetRef directly
+    const refPath = treePath.map(s => s.value).join('/');
+    const datasetRef = variant('value', { hash: valueHash, versions });
+    await storage.datasets.write(repo, ws, refPath, datasetRef);
+}
+/**
+ * Get the status of a single dataset at a path within a workspace.
+ *
+ * Returns the ref type, hash, East type, and size without downloading the value.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
+ * @param ws - Workspace name
+ * @param treePath - Path to the dataset
+ * @returns Dataset status including ref type, hash, type, and size
+ * @throws If workspace not deployed, path invalid, or path points to a tree
+ */
+export async function workspaceGetDatasetStatus(storage, repo, ws, treePath) {
+    if (treePath.length === 0) {
+        throw new Error('Cannot get dataset status at root path - root is always a tree');
+    }
+    // Validate path and get type from structure
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
     let currentStructure = rootStructure;
     for (let i = 0; i < treePath.length; i++) {
         const segment = treePath[i];
-        if (segment.type !== 'field') {
+        if (segment.type !== 'field')
             throw new Error(`Unsupported path segment type: ${segment.type}`);
-        }
-        if (currentStructure.type !== 'struct') {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
-        }
-        const childStructure = currentStructure.value.get(segment.value);
-        if (!childStructure) {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            const available = Array.from(currentStructure.value.keys()).join(', ');
-            throw new Error(`Field '${segment.value}' not found at '${pathSoFar}'. Available: ${available}`);
-        }
-        currentStructure = childStructure;
+        if (currentStructure.type !== 'struct')
+            throw new Error('Cannot descend into non-struct');
+        const child = currentStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found`);
+        currentStructure = child;
     }
-    // Final structure must be a value
     if (currentStructure.type !== 'value') {
         const pathStr = treePath.map(s => s.value).join('.');
         throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
     }
-    // Rebuild the tree path from leaf to root (structural sharing)
-    // Collect all tree hashes and structures along the path
-    const treeInfos = [];
-    let currentHash = state.rootHash;
-    currentStructure = rootStructure;
-    // Read all trees along the path (except the last segment which is the dataset)
-    for (let i = 0; i < treePath.length - 1; i++) {
-        treeInfos.push({ hash: currentHash, structure: currentStructure });
-        const segment = treePath[i];
-        const treeObject = await treeRead(repoPath, currentHash, currentStructure);
-        const childRef = treeObject[segment.value];
-        if (!childRef || childRef.type !== 'tree') {
-            throw new Error(`Expected tree ref at path segment ${i}`);
-        }
-        currentHash = childRef.value;
-        currentStructure = currentStructure.value.get(segment.value);
+    const datasetType = currentStructure.value.type;
+    // Read the ref file
+    const refPath = treePath.map(s => s.value).join('/');
+    const ref = await storage.datasets.read(repo, ws, refPath);
+    if (!ref || ref.type === 'unassigned') {
+        return { refType: 'unassigned', hash: null, datasetType, size: null };
+    }
+    if (ref.type === 'null') {
+        return { refType: 'null', hash: null, datasetType, size: 0 };
+    }
+    // value ref - get size from object store
+    const { size } = await storage.objects.stat(repo, ref.value.hash);
+    return { refType: 'value', hash: ref.value.hash, datasetType, size };
+}
+/**
+ * Check if a structure represents a task (has function_ir and output).
+ */
+function isTaskStructure(structure) {
+    if (structure.type !== 'struct')
+        return false;
+    const fields = structure.value;
+    return fields.has('function_ir') && fields.has('output');
+}
+/**
+ * Get the output type from a task structure (from structure, not value).
+ */
+function getTaskOutputTypeFromStructure(structure) {
+    if (structure.type !== 'struct')
+        return undefined;
+    const outputStructure = structure.value.get('output');
+    if (outputStructure?.type === 'value') {
+        return outputStructure.value.type;
+    }
+    return undefined;
+}
+/**
+ * Get the full tree structure at a path within a workspace.
+ *
+ * Recursively walks the structure and ref files to build a hierarchical
+ * structure suitable for display. Tasks are shown as leaves with their output type.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
+ * @param ws - Workspace name
+ * @param treePath - Path to start from (empty for root)
+ * @param options - Optional settings for depth limit and type inclusion
+ * @returns Array of tree nodes at the path
+ * @throws If workspace not deployed or path invalid
+ */
+export async function workspaceGetTree(storage, repo, ws, treePath, options = {}) {
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
+    const { maxDepth, includeTypes, includeStatus } = options;
+    // Navigate to the target structure
+    let targetStructure = rootStructure;
+    let pathPrefix = '';
+    for (const segment of treePath) {
+        if (segment.type !== 'field')
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        if (targetStructure.type !== 'struct')
+            throw new Error('Cannot descend into non-struct');
+        const child = targetStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found`);
+        pathPrefix = pathPrefix ? `${pathPrefix}/${segment.value}` : segment.value;
+        targetStructure = child;
+    }
+    if (targetStructure.type !== 'struct') {
+        const pathStr = treePath.map(s => s.value).join('.');
+        throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
     }
-    // Add the final tree that contains the dataset
-    treeInfos.push({ hash: currentHash, structure: currentStructure });
-    // Now rebuild from leaf to root
-    // Start with the provided value hash as the new ref
-    let newRef = { type: 'value', value: valueHash };
-    for (let i = treeInfos.length - 1; i >= 0; i--) {
-        const { hash, structure } = treeInfos[i];
-        const fieldName = treePath[i].value;
-        // Read the current tree
-        const treeObject = await treeRead(repoPath, hash, structure);
-        // Create modified tree with the new ref
-        const newTreeObject = {
-            ...treeObject,
-            [fieldName]: newRef,
-        };
-        // Write the new tree
-        const newTreeHash = await treeWrite(repoPath, newTreeObject, structure);
-        // This becomes the new ref for the parent
-        newRef = { type: 'tree', value: newTreeHash };
+    return walkStructure(storage, repo, ws, targetStructure, pathPrefix, 0, maxDepth, includeTypes, includeStatus);
+}
+/**
+ * Recursively walk structure and build TreeNode array using ref files.
+ */
+async function walkStructure(storage, repo, ws, structure, pathPrefix, currentDepth, maxDepth, includeTypes, includeStatus) {
+    if (structure.type !== 'struct') {
+        throw new Error('Expected struct structure for tree walk');
+    }
+    const entries = Array.from(structure.value.entries());
+    const nodes = await Promise.all(entries.map(async ([fieldName, childStructure]) => {
+        const childPath = pathPrefix ? `${pathPrefix}/${fieldName}` : fieldName;
+        if (childStructure.type === 'value') {
+            // Dataset (leaf node)
+            const node = {
+                name: fieldName,
+                kind: 'dataset',
+                datasetType: includeTypes ? childStructure.value.type : undefined,
+            };
+            if (includeStatus) {
+                const ref = await storage.datasets.read(repo, ws, childPath);
+                if (!ref || ref.type === 'unassigned') {
+                    node.refType = 'unassigned';
+                }
+                else if (ref.type === 'null') {
+                    node.refType = 'null';
+                    node.size = 0;
+                }
+                else {
+                    node.refType = 'value';
+                    node.hash = ref.value.hash;
+                    const { size } = await storage.objects.stat(repo, ref.value.hash);
+                    node.size = size;
+                }
+            }
+            return node;
+        }
+        // childStructure.type === 'struct'
+        // Task subtree — show as leaf with output type
+        if (isTaskStructure(childStructure)) {
+            const node = {
+                name: fieldName,
+                kind: 'dataset',
+                datasetType: includeTypes ? getTaskOutputTypeFromStructure(childStructure) : undefined,
+            };
+            if (includeStatus) {
+                // Read the output ref for the task
+                const outputRefPath = `${childPath}/output`;
+                const outputRef = await storage.datasets.read(repo, ws, outputRefPath);
+                if (!outputRef || outputRef.type === 'unassigned') {
+                    node.refType = 'unassigned';
+                }
+                else if (outputRef.type === 'null') {
+                    node.refType = 'null';
+                    node.size = 0;
+                }
+                else {
+                    node.refType = 'value';
+                    node.hash = outputRef.value.hash;
+                    const { size } = await storage.objects.stat(repo, outputRef.value.hash);
+                    node.size = size;
+                }
+            }
+            return node;
+        }
+        // Regular subtree
+        let children = [];
+        if (maxDepth === undefined || currentDepth < maxDepth) {
+            children = await walkStructure(storage, repo, ws, childStructure, childPath, currentDepth + 1, maxDepth, includeTypes, includeStatus);
+        }
+        return { name: fieldName, kind: 'tree', children };
+    }));
+    // Sort alphabetically for consistent output
+    nodes.sort((a, b) => a.name.localeCompare(b.name));
+    return nodes;
+}
+// =============================================================================
+// Package Operations (still use tree traversal for compatibility)
+// =============================================================================
+/**
+ * List field names at a tree path within a package's data tree.
+ *
+ * Note: In the new format, packages store per-dataset refs in data/ dir
+ * rather than tree objects. This function uses the structure directly.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
+ * @param name - Package name
+ * @param version - Package version
+ * @param path - Path to the tree node
+ * @returns Array of field names at the path
+ * @throws If package not found, path invalid, or path points to a dataset
+ */
+export async function packageListTree(storage, repo, name, version, path) {
+    const pkg = await packageRead(storage, repo, name, version);
+    const rootStructure = pkg.data.structure;
+    let currentStructure = rootStructure;
+    for (let i = 0; i < path.length; i++) {
+        const segment = path[i];
+        if (segment.type !== 'field')
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        if (currentStructure.type !== 'struct')
+            throw new Error('Path points to a dataset, not a tree');
+        const child = currentStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found`);
+        currentStructure = child;
     }
-    // The final newRef is always a tree ref pointing to the new root
-    if (newRef.type !== 'tree' || newRef.value === null) {
-        throw new Error('Internal error: expected tree ref after rebuilding path');
+    if (currentStructure.type !== 'struct') {
+        throw new Error('Path points to a dataset, not a tree');
     }
-    const newRootHash = newRef.value;
-    // Update workspace state atomically
-    await writeWorkspaceState(repoPath, ws, {
-        ...state,
-        rootHash: newRootHash,
-        rootUpdatedAt: new Date(),
-    });
+    return Array.from(currentStructure.value.keys());
 }
 //# sourceMappingURL=trees.js.map