@elaraai/e3-core 0.0.2-beta.35 → 0.0.2-beta.37

package/dist/src/trees.js

@@ -12,8 +12,11 @@
  * tree node. This enables proper encoding/decoding with the correct StructType
  * and supports future tree types (array, variant, etc.).
  *
- * Low-level operations work with hashes directly (by-hash).
- * High-level operations traverse paths from a root (by-path).
+ * Workspace operations use per-dataset ref files (DatasetRef) instead of tree
+ * traversal. This enables concurrent writes without serialization.
+ *
+ * Low-level tree read/write operations remain for computing root hashes and
+ * for package operations.
  */
 import { decodeBeast2, decodeBeast2For, encodeBeast2For, StructType, variant, } from '@elaraai/east';
 import { DataRefType, PackageObjectType, WorkspaceStateType } from '@elaraai/e3-types';
@@ -109,152 +112,11 @@ export async function datasetWrite(storage, repo, value, type) {
     const data = encoder(value);
     return storage.objects.write(repo, data);
 }
-/**
- * Traverse a tree from root to a path, co-walking structure and data.
- *
- * @param storage - Storage backend
- * @param repo - Repository identifier
- * @param rootHash - Hash of the root tree object
- * @param rootStructure - Structure of the root tree
- * @param path - Path to traverse
- * @returns The structure and DataRef at the path location
- * @throws If path is invalid or traversal fails
- */
-async function traverse(storage, repo, rootHash, rootStructure, path) {
-    let currentStructure = rootStructure;
-    let currentHash = rootHash;
-    for (let i = 0; i < path.length; i++) {
-        const segment = path[i];
-        if (segment.type !== 'field') {
-            throw new Error(`Unsupported path segment type: ${segment.type}`);
-        }
-        const fieldName = segment.value;
-        // Current structure must be a struct tree to descend into
-        if (currentStructure.type !== 'struct') {
-            const pathSoFar = path.slice(0, i).map(s => s.value).join('.');
-            throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
-        }
-        // Read the current tree object
-        const treeObject = await treeRead(storage, repo, currentHash, currentStructure);
-        // Look up the child ref
-        const childRef = treeObject[fieldName];
-        if (!childRef) {
-            const pathSoFar = path.slice(0, i).map(s => s.value).join('.');
-            const available = Object.keys(treeObject).join(', ');
-            throw new Error(`Field '${fieldName}' not found at '${pathSoFar}'. Available: ${available}`);
-        }
-        // Look up the child structure
-        const childStructure = currentStructure.value.get(fieldName);
-        if (!childStructure) {
-            throw new Error(`Field '${fieldName}' not found in structure`);
-        }
-        // If this is the last segment, return the result
-        if (i === path.length - 1) {
-            return { structure: childStructure, ref: childRef };
-        }
-        // Otherwise, continue traversing (must be a tree ref)
-        if (childRef.type !== 'tree') {
-            const pathSoFar = path.slice(0, i + 1).map(s => s.value).join('.');
-            throw new Error(`Expected tree ref at '${pathSoFar}', got '${childRef.type}'`);
-        }
-        currentStructure = childStructure;
-        currentHash = childRef.value;
-    }
-    // Empty path - return root
-    return {
-        structure: rootStructure,
-        ref: { type: 'tree', value: rootHash },
-    };
-}
-/**
- * List field names at a tree path within a package's data tree.
- *
- * @param storage - Storage backend
- * @param repo - Repository identifier
- * @param name - Package name
- * @param version - Package version
- * @param path - Path to the tree node
- * @returns Array of field names at the path
- * @throws If package not found, path invalid, or path points to a dataset
- */
-export async function packageListTree(storage, repo, name, version, path) {
-    // Read the package to get root structure and hash
-    const pkg = await packageRead(storage, repo, name, version);
-    const rootStructure = pkg.data.structure;
-    const rootHash = pkg.data.value;
-    if (path.length === 0) {
-        // Empty path - list root tree fields
-        if (rootStructure.type !== 'struct') {
-            throw new Error('Root is not a tree');
-        }
-        const treeObject = await treeRead(storage, repo, rootHash, rootStructure);
-        return Object.keys(treeObject);
-    }
-    // Traverse to the path
-    const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, path);
-    // Must be a tree structure
-    if (structure.type !== 'struct') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
-    }
-    // Must be a tree ref
-    if (ref.type !== 'tree') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
-    }
-    // Read the tree and return field names
-    const treeObject = await treeRead(storage, repo, ref.value, structure);
-    return Object.keys(treeObject);
-}
-/**
- * Read and decode a dataset value at a path within a package's data tree.
- *
- * @param storage - Storage backend
- * @param repo - Repository identifier
- * @param name - Package name
- * @param version - Package version
- * @param path - Path to the dataset
- * @returns The decoded dataset value
- * @throws If package not found, path invalid, or path points to a tree
- */
-export async function packageGetDataset(storage, repo, name, version, path) {
-    // Read the package to get root structure and hash
-    const pkg = await packageRead(storage, repo, name, version);
-    const rootStructure = pkg.data.structure;
-    const rootHash = pkg.data.value;
-    if (path.length === 0) {
-        throw new Error('Cannot get dataset at root path - root is always a tree');
-    }
-    // Traverse to the path
-    const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, path);
-    // Must be a value structure
-    if (structure.type !== 'value') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
-    }
-    // Handle different ref types
-    if (ref.type === 'unassigned') {
-        throw new Error(`Dataset at path is unassigned (pending task output)`);
-    }
-    if (ref.type === 'null') {
-        return null;
-    }
-    if (ref.type === 'tree') {
-        const pathStr = path.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
-    }
-    // Read and return the dataset value
-    const result = await datasetRead(storage, repo, ref.value);
-    return result.value;
-}
 /**
  * Update a dataset at a path within a workspace.
  *
- * This creates new tree objects along the path with structural sharing,
- * then atomically updates the workspace root.
- *
- * Acquires an exclusive lock on the workspace for the duration of the write
- * to prevent concurrent modifications.
+ * Writes the value to the object store and updates the per-dataset ref file.
+ * Uses shared structure lock to allow concurrent writes.
  *
  * @param storage - Storage backend
  * @param repo - Repository identifier
@@ -274,7 +136,7 @@ export async function workspaceSetDataset(storage, repo, ws, treePath, value, ty
     const externalLock = options.lock;
     let lock = externalLock ?? null;
     if (!lock) {
-        lock = await storage.locks.acquire(repo, ws, variant('dataset_write', null));
+        lock = await storage.locks.acquire(repo, ws, variant('dataset_write', null), { mode: 'shared' });
         if (!lock) {
             const state = await storage.locks.getState(repo, ws);
             throw new WorkspaceLockError(ws, state ? {
@@ -284,7 +146,48 @@ export async function workspaceSetDataset(storage, repo, ws, treePath, value, ty
         }
     }
     try {
-        await workspaceSetDatasetUnlocked(storage, repo, ws, treePath, value, type);
+        const wsState = await readWorkspaceState(storage, repo, ws);
+        // Read the deployed package object to get the structure
+        const pkgData = await storage.objects.read(repo, wsState.packageHash);
+        const decoder = decodeBeast2For(PackageObjectType);
+        const pkgObject = decoder(Buffer.from(pkgData));
+        const rootStructure = pkgObject.data.structure;
+        // Validate that the path leads to a value structure and check writable
+        let currentStructure = rootStructure;
+        for (let i = 0; i < treePath.length; i++) {
+            const segment = treePath[i];
+            if (segment.type !== 'field') {
+                throw new Error(`Unsupported path segment type: ${segment.type}`);
+            }
+            if (currentStructure.type !== 'struct') {
+                const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
+                throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
+            }
+            const childStructure = currentStructure.value.get(segment.value);
+            if (!childStructure) {
+                const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
+                const available = Array.from(currentStructure.value.keys()).join(', ');
+                throw new Error(`Field '${segment.value}' not found at '${pathSoFar}'. Available: ${available}`);
+            }
+            currentStructure = childStructure;
+        }
+        // Final structure must be a value
+        if (currentStructure.type !== 'value') {
+            const pathStr = treePath.map(s => s.value).join('.');
+            throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
+        }
+        // Check writable flag
+        if (!currentStructure.value.writable) {
+            const pathStr = treePath.map(s => s.value).join('.');
+            throw new Error(`Dataset at '${pathStr}' is not writable`);
+        }
+        // Write the new dataset value to object store
+        const newValueHash = await datasetWrite(storage, repo, value, type);
+        // Build ref path from tree path
+        const refPath = treePath.map(s => s.value).join('/');
+        // Write the DatasetRef with empty version vector (will be populated by dataflow)
+        const datasetRef = variant('value', { hash: newValueHash, versions: new Map() });
+        await storage.datasets.write(repo, ws, refPath, datasetRef);
     }
     finally {
         // Only release the lock if we acquired it internally
@@ -293,105 +196,9 @@ export async function workspaceSetDataset(storage, repo, ws, treePath, value, ty
         }
     }
 }
-/**
- * Internal: Update a dataset without acquiring a lock.
- * Caller must hold the workspace lock.
- */
-async function workspaceSetDatasetUnlocked(storage, repo, ws, treePath, value, type) {
-    const state = await readWorkspaceState(storage, repo, ws);
-    // Read the deployed package object to get the structure
-    const pkgData = await storage.objects.read(repo, state.packageHash);
-    const decoder = decodeBeast2For(PackageObjectType);
-    const pkgObject = decoder(Buffer.from(pkgData));
-    const rootStructure = pkgObject.data.structure;
-    // Validate that the path leads to a value structure
-    let currentStructure = rootStructure;
-    for (let i = 0; i < treePath.length; i++) {
-        const segment = treePath[i];
-        if (segment.type !== 'field') {
-            throw new Error(`Unsupported path segment type: ${segment.type}`);
-        }
-        if (currentStructure.type !== 'struct') {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
-        }
-        const childStructure = currentStructure.value.get(segment.value);
-        if (!childStructure) {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            const available = Array.from(currentStructure.value.keys()).join(', ');
-            throw new Error(`Field '${segment.value}' not found at '${pathSoFar}'. Available: ${available}`);
-        }
-        currentStructure = childStructure;
-    }
-    // Final structure must be a value
-    if (currentStructure.type !== 'value') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
-    }
-    // Write the new dataset value
-    const newValueHash = await datasetWrite(storage, repo, value, type);
-    // Now rebuild the tree path from leaf to root (structural sharing)
-    // We need to read each tree along the path, modify it, and write a new version
-    // Collect all tree hashes and structures along the path
-    const treeInfos = [];
-    let currentHash = state.rootHash;
-    currentStructure = rootStructure;
-    // Read all trees along the path (except the last segment which is the dataset)
-    for (let i = 0; i < treePath.length - 1; i++) {
-        treeInfos.push({ hash: currentHash, structure: currentStructure });
-        const segment = treePath[i];
-        const treeObject = await treeRead(storage, repo, currentHash, currentStructure);
-        const childRef = treeObject[segment.value];
-        if (!childRef || childRef.type !== 'tree') {
-            throw new Error(`Expected tree ref at path segment ${i}`);
-        }
-        currentHash = childRef.value;
-        currentStructure = currentStructure.value.get(segment.value);
-    }
-    // Add the final tree that contains the dataset
-    treeInfos.push({ hash: currentHash, structure: currentStructure });
-    // Now rebuild from leaf to root
-    // Start with the new value hash as the new ref
-    let newRef = { type: 'value', value: newValueHash };
-    for (let i = treeInfos.length - 1; i >= 0; i--) {
-        const { hash, structure } = treeInfos[i];
-        const fieldName = treePath[i].value;
-        // Read the current tree
-        const treeObject = await treeRead(storage, repo, hash, structure);
-        // Create modified tree with the new ref
-        const newTreeObject = {
-            ...treeObject,
-            [fieldName]: newRef,
-        };
-        // Write the new tree
-        const newTreeHash = await treeWrite(storage, repo, newTreeObject, structure);
-        // This becomes the new ref for the parent
-        newRef = { type: 'tree', value: newTreeHash };
-    }
-    // The final newRef is always a tree ref pointing to the new root
-    // (because we start with a value ref and wrap it in tree refs bottom-up)
-    if (newRef.type !== 'tree' || newRef.value === null) {
-        throw new Error('Internal error: expected tree ref after rebuilding path');
-    }
-    const newRootHash = newRef.value;
-    // Update workspace state atomically
-    await writeWorkspaceState(storage, repo, ws, {
-        ...state,
-        rootHash: newRootHash,
-        rootUpdatedAt: new Date(),
-    });
-}
 // =============================================================================
 // Workspace Helper Functions
 // =============================================================================
-/**
- * Write workspace state to file atomically.
- */
-async function writeWorkspaceState(storage, repo, ws, state) {
-    const encoder = encodeBeast2For(WorkspaceStateType);
-    const data = encoder(state);
-    await storage.refs.workspaceWrite(repo, ws, data);
-}
 /**
  * Read workspace state from file.
  * @throws {WorkspaceNotFoundError} If workspace doesn't exist
@@ -409,101 +216,110 @@ async function readWorkspaceState(storage, repo, ws) {
     return decoder(data);
 }
 /**
- * Get root structure and hash for a workspace.
+ * Get root structure for a workspace.
  * Reads the deployed package object to get the structure.
  */
-async function getWorkspaceRootInfo(storage, repo, ws) {
-    const state = await readWorkspaceState(storage, repo, ws);
+async function getWorkspaceStructure(storage, repo, ws) {
+    const wsState = await readWorkspaceState(storage, repo, ws);
     // Read the deployed package object using the stored hash
-    const pkgData = await storage.objects.read(repo, state.packageHash);
+    const pkgData = await storage.objects.read(repo, wsState.packageHash);
     const decoder = decodeBeast2For(PackageObjectType);
     const pkgObject = decoder(Buffer.from(pkgData));
     return {
-        rootHash: state.rootHash,
         rootStructure: pkgObject.data.structure,
     };
 }
 // =============================================================================
-// Workspace High-level Operations (by path)
+// Workspace High-level Operations (by path) - Using per-dataset refs
 // =============================================================================
 /**
  * List field names at a tree path within a workspace's data tree.
  *
+ * Uses the structure to determine available fields (no tree traversal needed).
+ *
  * @param storage - Storage backend
  * @param repo - Repository identifier
  * @param ws - Workspace name
- * @param path - Path to the tree node
+ * @param treePath - Path to the tree node
  * @returns Array of field names at the path
  * @throws If workspace not deployed, path invalid, or path points to a dataset
  */
 export async function workspaceListTree(storage, repo, ws, treePath) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
-    if (treePath.length === 0) {
-        // Empty path - list root tree fields
-        if (rootStructure.type !== 'struct') {
-            throw new Error('Root is not a tree');
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
+    // Navigate structure to find the target node
+    let currentStructure = rootStructure;
+    for (let i = 0; i < treePath.length; i++) {
+        const segment = treePath[i];
+        if (segment.type !== 'field') {
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        }
+        if (currentStructure.type !== 'struct') {
+            const pathStr = treePath.slice(0, i).map(s => s.value).join('.');
+            throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
+        }
+        const childStructure = currentStructure.value.get(segment.value);
+        if (!childStructure) {
+            throw new Error(`Field '${segment.value}' not found in structure`);
         }
-        const treeObject = await treeRead(storage, repo, rootHash, rootStructure);
-        return Object.keys(treeObject);
+        currentStructure = childStructure;
     }
-    // Traverse to the path
-    const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
-    // Must be a tree structure
-    if (structure.type !== 'struct') {
+    if (currentStructure.type !== 'struct') {
         const pathStr = treePath.map(s => s.value).join('.');
         throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
     }
-    // Must be a tree ref
-    if (ref.type !== 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
-    }
-    // Read the tree and return field names
-    const treeObject = await treeRead(storage, repo, ref.value, structure);
-    return Object.keys(treeObject);
+    return Array.from(currentStructure.value.keys());
 }
 /**
  * Read and decode a dataset value at a path within a workspace's data tree.
  *
+ * Reads the per-dataset ref file to get the value hash, then decodes the value.
+ *
  * @param storage - Storage backend
  * @param repo - Repository identifier
  * @param ws - Workspace name
- * @param path - Path to the dataset
+ * @param treePath - Path to the dataset
  * @returns The decoded dataset value
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
 export async function workspaceGetDataset(storage, repo, ws, treePath) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
     if (treePath.length === 0) {
         throw new Error('Cannot get dataset at root path - root is always a tree');
     }
-    // Traverse to the path
-    const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
-    // Must be a value structure
-    if (structure.type !== 'value') {
+    // Validate path against structure
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
+    let currentStructure = rootStructure;
+    for (let i = 0; i < treePath.length; i++) {
+        const segment = treePath[i];
+        if (segment.type !== 'field')
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        if (currentStructure.type !== 'struct')
+            throw new Error(`Cannot descend into non-struct`);
+        const child = currentStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found in structure`);
+        currentStructure = child;
+    }
+    if (currentStructure.type !== 'value') {
         const pathStr = treePath.map(s => s.value).join('.');
         throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
     }
-    // Handle different ref types
-    if (ref.type === 'unassigned') {
+    // Read the ref file
+    const refPath = treePath.map(s => s.value).join('/');
+    const ref = await storage.datasets.read(repo, ws, refPath);
+    if (!ref || ref.type === 'unassigned') {
         throw new Error(`Dataset at path is unassigned (pending task output)`);
     }
     if (ref.type === 'null') {
         return null;
     }
-    if (ref.type === 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
-    }
     // Read and return the dataset value
-    const result = await datasetRead(storage, repo, ref.value);
+    const result = await datasetRead(storage, repo, ref.value.hash);
     return result.value;
 }
 /**
  * Get the hash of a dataset at a path within a workspace's data tree.
  *
- * Unlike workspaceGetDataset which decodes the value, this returns the raw
- * hash reference. Useful for dataflow execution which operates on hashes.
+ * Reads the per-dataset ref file directly. No tree traversal needed.
  *
  * @param storage - Storage backend
  * @param repo - Repository identifier
@@ -513,35 +329,27 @@ export async function workspaceGetDataset(storage, repo, ws, treePath) {
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
 export async function workspaceGetDatasetHash(storage, repo, ws, treePath) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
     if (treePath.length === 0) {
         throw new Error('Cannot get dataset at root path - root is always a tree');
     }
-    // Traverse to the path
-    const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
-    // Must be a value structure
-    if (structure.type !== 'value') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
+    // Read the ref file directly using the path
+    const refPath = treePath.map(s => s.value).join('/');
+    const ref = await storage.datasets.read(repo, ws, refPath);
+    if (!ref || ref.type === 'unassigned') {
+        return { refType: 'unassigned', hash: null };
     }
-    // Return ref type and hash
-    if (ref.type === 'unassigned' || ref.type === 'null') {
-        return { refType: ref.type, hash: null };
-    }
-    if (ref.type === 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
+    if (ref.type === 'null') {
+        return { refType: 'null', hash: null };
     }
-    return { refType: ref.type, hash: ref.value };
+    return { refType: 'value', hash: ref.value.hash };
 }
 /**
  * Set a dataset at a path within a workspace using a pre-computed hash.
  *
- * Unlike workspaceSetDataset which encodes a value, this takes a hash
- * directly. Useful for dataflow execution which already has the output hash.
+ * Writes a DatasetRef file directly. No tree path-copy needed.
  *
  * IMPORTANT: This function does NOT acquire a workspace lock. The caller must
- * hold an exclusive lock on the workspace before calling this function. This
+ * hold a lock on the workspace before calling this function. This
  * is typically used by dataflowExecute which holds the lock for the entire
  * execution.
  *
@@ -550,92 +358,16 @@ export async function workspaceGetDatasetHash(storage, repo, ws, treePath) {
  * @param ws - Workspace name
  * @param treePath - Path to the dataset
  * @param valueHash - Hash of the dataset value already in the object store
- * @returns The new root hash after updating the tree
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
-export async function workspaceSetDatasetByHash(storage, repo, ws, treePath, valueHash) {
+export async function workspaceSetDatasetByHash(storage, repo, ws, treePath, valueHash, versions) {
     if (treePath.length === 0) {
         throw new Error('Cannot set dataset at root path - root is always a tree');
     }
-    const state = await readWorkspaceState(storage, repo, ws);
-    // Read the deployed package object to get the structure
-    const pkgData = await storage.objects.read(repo, state.packageHash);
-    const decoder = decodeBeast2For(PackageObjectType);
-    const pkgObject = decoder(Buffer.from(pkgData));
-    const rootStructure = pkgObject.data.structure;
-    // Validate that the path leads to a value structure
-    let currentStructure = rootStructure;
-    for (let i = 0; i < treePath.length; i++) {
-        const segment = treePath[i];
-        if (segment.type !== 'field') {
-            throw new Error(`Unsupported path segment type: ${segment.type}`);
-        }
-        if (currentStructure.type !== 'struct') {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            throw new Error(`Cannot descend into non-struct at path '${pathSoFar}'`);
-        }
-        const childStructure = currentStructure.value.get(segment.value);
-        if (!childStructure) {
-            const pathSoFar = treePath.slice(0, i).map(s => s.value).join('.');
-            const available = Array.from(currentStructure.value.keys()).join(', ');
-            throw new Error(`Field '${segment.value}' not found at '${pathSoFar}'. Available: ${available}`);
-        }
-        currentStructure = childStructure;
-    }
-    // Final structure must be a value
-    if (currentStructure.type !== 'value') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
-    }
-    // Rebuild the tree path from leaf to root (structural sharing)
-    // Collect all tree hashes and structures along the path
-    const treeInfos = [];
-    let currentHash = state.rootHash;
-    currentStructure = rootStructure;
-    // Read all trees along the path (except the last segment which is the dataset)
-    for (let i = 0; i < treePath.length - 1; i++) {
-        treeInfos.push({ hash: currentHash, structure: currentStructure });
-        const segment = treePath[i];
-        const treeObject = await treeRead(storage, repo, currentHash, currentStructure);
-        const childRef = treeObject[segment.value];
-        if (!childRef || childRef.type !== 'tree') {
-            throw new Error(`Expected tree ref at path segment ${i}`);
-        }
-        currentHash = childRef.value;
-        currentStructure = currentStructure.value.get(segment.value);
-    }
-    // Add the final tree that contains the dataset
-    treeInfos.push({ hash: currentHash, structure: currentStructure });
-    // Now rebuild from leaf to root
-    // Start with the provided value hash as the new ref
-    let newRef = { type: 'value', value: valueHash };
-    for (let i = treeInfos.length - 1; i >= 0; i--) {
-        const { hash, structure } = treeInfos[i];
-        const fieldName = treePath[i].value;
-        // Read the current tree
-        const treeObject = await treeRead(storage, repo, hash, structure);
-        // Create modified tree with the new ref
-        const newTreeObject = {
-            ...treeObject,
-            [fieldName]: newRef,
-        };
-        // Write the new tree
-        const newTreeHash = await treeWrite(storage, repo, newTreeObject, structure);
-        // This becomes the new ref for the parent
-        newRef = { type: 'tree', value: newTreeHash };
-    }
-    // The final newRef is always a tree ref pointing to the new root
-    if (newRef.type !== 'tree' || newRef.value === null) {
-        throw new Error('Internal error: expected tree ref after rebuilding path');
-    }
-    const newRootHash = newRef.value;
-    // Update workspace state atomically
-    await writeWorkspaceState(storage, repo, ws, {
-        ...state,
-        rootHash: newRootHash,
-        rootUpdatedAt: new Date(),
-    });
-    return newRootHash;
+    // Write the DatasetRef directly
+    const refPath = treePath.map(s => s.value).join('/');
+    const datasetRef = variant('value', { hash: valueHash, versions });
+    await storage.datasets.write(repo, ws, refPath, datasetRef);
 }
 /**
  * Get the status of a single dataset at a path within a workspace.
@@ -650,32 +382,40 @@ export async function workspaceSetDatasetByHash(storage, repo, ws, treePath, val
  * @throws If workspace not deployed, path invalid, or path points to a tree
  */
 export async function workspaceGetDatasetStatus(storage, repo, ws, treePath) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
     if (treePath.length === 0) {
         throw new Error('Cannot get dataset status at root path - root is always a tree');
     }
-    // Traverse to the path
-    const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
-    // Must be a value structure
-    if (structure.type !== 'value') {
+    // Validate path and get type from structure
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
+    let currentStructure = rootStructure;
+    for (let i = 0; i < treePath.length; i++) {
+        const segment = treePath[i];
+        if (segment.type !== 'field')
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        if (currentStructure.type !== 'struct')
+            throw new Error('Cannot descend into non-struct');
+        const child = currentStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found`);
+        currentStructure = child;
+    }
+    if (currentStructure.type !== 'value') {
         const pathStr = treePath.map(s => s.value).join('.');
         throw new Error(`Path '${pathStr}' points to a tree, not a dataset`);
     }
-    const datasetType = structure.value;
-    // Handle different ref types
-    if (ref.type === 'unassigned') {
+    const datasetType = currentStructure.value.type;
+    // Read the ref file
+    const refPath = treePath.map(s => s.value).join('/');
+    const ref = await storage.datasets.read(repo, ws, refPath);
+    if (!ref || ref.type === 'unassigned') {
         return { refType: 'unassigned', hash: null, datasetType, size: null };
     }
     if (ref.type === 'null') {
         return { refType: 'null', hash: null, datasetType, size: 0 };
     }
-    if (ref.type === 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' structure says value but ref is tree`);
-    }
     // value ref - get size from object store
-    const { size } = await storage.objects.stat(repo, ref.value);
-    return { refType: 'value', hash: ref.value, datasetType, size };
+    const { size } = await storage.objects.stat(repo, ref.value.hash);
+    return { refType: 'value', hash: ref.value.hash, datasetType, size };
 }
 /**
  * Check if a structure represents a task (has function_ir and output).
@@ -694,15 +434,15 @@ function getTaskOutputTypeFromStructure(structure) {
         return undefined;
     const outputStructure = structure.value.get('output');
     if (outputStructure?.type === 'value') {
-        return outputStructure.value;
+        return outputStructure.value.type;
     }
     return undefined;
 }
 /**
  * Get the full tree structure at a path within a workspace.
  *
- * Recursively walks the tree and returns a hierarchical structure
- * suitable for display. Tasks are shown as leaves with their output type.
+ * Recursively walks the structure and ref files to build a hierarchical
+ * structure suitable for display. Tasks are shown as leaves with their output type.
  *
  * @param storage - Storage backend
  * @param repo - Repository identifier
@@ -713,83 +453,87 @@ function getTaskOutputTypeFromStructure(structure) {
  * @throws If workspace not deployed or path invalid
  */
 export async function workspaceGetTree(storage, repo, ws, treePath, options = {}) {
-    const { rootHash, rootStructure } = await getWorkspaceRootInfo(storage, repo, ws);
+    const { rootStructure } = await getWorkspaceStructure(storage, repo, ws);
     const { maxDepth, includeTypes, includeStatus } = options;
-    // If path is empty, start from root
-    if (treePath.length === 0) {
-        if (rootStructure.type !== 'struct') {
-            throw new Error('Root is not a tree');
-        }
-        return walkTree(storage, repo, rootHash, rootStructure, 0, maxDepth, includeTypes, includeStatus);
-    }
-    // Traverse to the path first
-    const { structure, ref } = await traverse(storage, repo, rootHash, rootStructure, treePath);
-    // Must be a tree structure
-    if (structure.type !== 'struct') {
+    // Navigate to the target structure
+    let targetStructure = rootStructure;
+    let pathPrefix = '';
+    for (const segment of treePath) {
+        if (segment.type !== 'field')
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        if (targetStructure.type !== 'struct')
+            throw new Error('Cannot descend into non-struct');
+        const child = targetStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found`);
+        pathPrefix = pathPrefix ? `${pathPrefix}/${segment.value}` : segment.value;
+        targetStructure = child;
+    }
+    if (targetStructure.type !== 'struct') {
         const pathStr = treePath.map(s => s.value).join('.');
         throw new Error(`Path '${pathStr}' points to a dataset, not a tree`);
     }
-    // Must be a tree ref
-    if (ref.type !== 'tree') {
-        const pathStr = treePath.map(s => s.value).join('.');
-        throw new Error(`Path '${pathStr}' has ref type '${ref.type}', expected 'tree'`);
-    }
-    return walkTree(storage, repo, ref.value, structure, 0, maxDepth, includeTypes, includeStatus);
+    return walkStructure(storage, repo, ws, targetStructure, pathPrefix, 0, maxDepth, includeTypes, includeStatus);
 }
 /**
- * Recursively walk a tree and build TreeNode array.
+ * Recursively walk structure and build TreeNode array using ref files.
  */
-async function walkTree(storage, repo, treeHash, structure, currentDepth, maxDepth, includeTypes, includeStatus) {
+async function walkStructure(storage, repo, ws, structure, pathPrefix, currentDepth, maxDepth, includeTypes, includeStatus) {
     if (structure.type !== 'struct') {
         throw new Error('Expected struct structure for tree walk');
     }
-    const treeObject = await treeRead(storage, repo, treeHash, structure);
-    // Filter to fields that exist in the structure, then process in parallel
-    const entries = Object.entries(treeObject)
-        .filter(([fieldName]) => structure.value.has(fieldName));
-    const nodes = await Promise.all(entries.map(async ([fieldName, childRef]) => {
-        const childStructure = structure.value.get(fieldName);
+    const entries = Array.from(structure.value.entries());
+    const nodes = await Promise.all(entries.map(async ([fieldName, childStructure]) => {
+        const childPath = pathPrefix ? `${pathPrefix}/${fieldName}` : fieldName;
         if (childStructure.type === 'value') {
             // Dataset (leaf node)
             const node = {
                 name: fieldName,
                 kind: 'dataset',
-                datasetType: includeTypes ? childStructure.value : undefined,
+                datasetType: includeTypes ? childStructure.value.type : undefined,
             };
             if (includeStatus) {
-                node.refType = childRef.type;
-                if (childRef.type === 'value') {
-                    node.hash = childRef.value;
-                    const { size } = await storage.objects.stat(repo, childRef.value);
-                    node.size = size;
+                const ref = await storage.datasets.read(repo, ws, childPath);
+                if (!ref || ref.type === 'unassigned') {
+                    node.refType = 'unassigned';
                 }
-                else if (childRef.type === 'null') {
+                else if (ref.type === 'null') {
+                    node.refType = 'null';
                     node.size = 0;
                 }
+                else {
+                    node.refType = 'value';
+                    node.hash = ref.value.hash;
+                    const { size } = await storage.objects.stat(repo, ref.value.hash);
+                    node.size = size;
+                }
             }
             return node;
         }
         // childStructure.type === 'struct'
         // Task subtree — show as leaf with output type
-        if (isTaskStructure(childStructure) && childRef.type === 'tree') {
+        if (isTaskStructure(childStructure)) {
             const node = {
                 name: fieldName,
                 kind: 'dataset',
                 datasetType: includeTypes ? getTaskOutputTypeFromStructure(childStructure) : undefined,
             };
             if (includeStatus) {
-                const taskTree = await treeRead(storage, repo, childRef.value, childStructure);
-                const outputRef = taskTree['output'];
-                if (outputRef) {
-                    node.refType = outputRef.type;
-                    if (outputRef.type === 'value') {
-                        node.hash = outputRef.value;
-                        const { size } = await storage.objects.stat(repo, outputRef.value);
-                        node.size = size;
-                    }
-                    else if (outputRef.type === 'null') {
-                        node.size = 0;
-                    }
+                // Read the output ref for the task
+                const outputRefPath = `${childPath}/output`;
+                const outputRef = await storage.datasets.read(repo, ws, outputRefPath);
+                if (!outputRef || outputRef.type === 'unassigned') {
+                    node.refType = 'unassigned';
+                }
+                else if (outputRef.type === 'null') {
+                    node.refType = 'null';
+                    node.size = 0;
+                }
+                else {
+                    node.refType = 'value';
+                    node.hash = outputRef.value.hash;
+                    const { size } = await storage.objects.stat(repo, outputRef.value.hash);
+                    node.size = size;
                 }
             }
             return node;
@@ -797,9 +541,7 @@ async function walkTree(storage, repo, treeHash, structure, currentDepth, maxDep
         // Regular subtree
         let children = [];
         if (maxDepth === undefined || currentDepth < maxDepth) {
-            if (childRef.type === 'tree') {
-                children = await walkTree(storage, repo, childRef.value, childStructure, currentDepth + 1, maxDepth, includeTypes, includeStatus);
-            }
+            children = await walkStructure(storage, repo, ws, childStructure, childPath, currentDepth + 1, maxDepth, includeTypes, includeStatus);
         }
         return { name: fieldName, kind: 'tree', children };
     }));
@@ -807,4 +549,41 @@ async function walkTree(storage, repo, treeHash, structure, currentDepth, maxDep
     nodes.sort((a, b) => a.name.localeCompare(b.name));
     return nodes;
 }
+// =============================================================================
+// Package Operations (still use tree traversal for compatibility)
+// =============================================================================
+/**
+ * List field names at a tree path within a package's data tree.
+ *
+ * Note: In the new format, packages store per-dataset refs in data/ dir
+ * rather than tree objects. This function uses the structure directly.
+ *
+ * @param storage - Storage backend
+ * @param repo - Repository identifier
+ * @param name - Package name
+ * @param version - Package version
+ * @param path - Path to the tree node
+ * @returns Array of field names at the path
+ * @throws If package not found, path invalid, or path points to a dataset
+ */
+export async function packageListTree(storage, repo, name, version, path) {
+    const pkg = await packageRead(storage, repo, name, version);
+    const rootStructure = pkg.data.structure;
+    let currentStructure = rootStructure;
+    for (let i = 0; i < path.length; i++) {
+        const segment = path[i];
+        if (segment.type !== 'field')
+            throw new Error(`Unsupported path segment type: ${segment.type}`);
+        if (currentStructure.type !== 'struct')
+            throw new Error('Path points to a dataset, not a tree');
+        const child = currentStructure.value.get(segment.value);
+        if (!child)
+            throw new Error(`Field '${segment.value}' not found`);
+        currentStructure = child;
+    }
+    if (currentStructure.type !== 'struct') {
+        throw new Error('Path points to a dataset, not a tree');
+    }
+    return Array.from(currentStructure.value.keys());
+}
 //# sourceMappingURL=trees.js.map