@pdpp/cli 0.1.0-beta.2 → 0.1.0-beta.3

package/README.md

@@ -4,21 +4,64 @@ Command-line tools for PDPP providers.
 
 ## Status
 
-This package is the public npm home for the `pdpp` command. The beta CLI supports
-`pdpp connect <provider-url>` for delegated access: the agent runs the command,
-the owner approves scoped access in the browser, and the CLI stores scoped client
-credentials in the project-local `.pdpp/` cache without asking for an owner
-bearer token.
+This package is the public npm home for the `pdpp` command. The beta CLI
+supports three command namespaces:
+
+- **`pdpp connect <provider-url>`** — delegated access: discovers provider
+  metadata, self-registers a public client when the AS advertises dynamic
+  registration, asks the owner to approve scoped access in the browser, and
+  stores scoped client credentials in the project-local `.pdpp/` cache without
+  asking for an owner bearer token.
+
+- **`pdpp collector <advertise|enroll|run>`** — operator surface for the
+  local collector runner. Pairs a host the operator controls (Claude Code or
+  Codex CLI data) with a remote PDPP reference deployment via device-scoped
+  enrollment, then runs connectors that the provider/control-plane container
+  cannot run on its own. The runner ships separately as
+  `@pdpp/local-collector` and owns the `pdpp-local-collector` binary; `pdpp
+  collector ...` is a slim `@pdpp/cli` shim that resolves that package lazily.
+  Public onboarding should use `npx -y @pdpp/local-collector ...` or
+  `npm i -g @pdpp/local-collector` unless the operator intentionally wants the
+  `@pdpp/cli` shim.
+
+- **`pdpp ref ...`** — reference operator diagnostics over `_ref` routes on a
+  running reference deployment. Current subcommands: `pdpp ref run timeline
+  <run-id>`, `pdpp ref grant timeline <grant-id>`, `pdpp ref trace show
+  <trace-id>`. Requires `PDPP_OWNER_SESSION_COOKIE` when owner auth is enabled.
+  These are reference-only operator tools, not core PDPP protocol.
 
 ## Install
 
 ```bash
+# @pdpp/cli package, npx-launched pdpp binary
 npx -y @pdpp/cli@beta --help
 ```
 
 Use the `beta` dist-tag until PDPP intentionally enables stable `latest`
 publication.
 
+When working from this monorepo without installing or linking the binary, use
+the workspace executable:
+
+```bash
+# @pdpp/cli package, workspace-launched pdpp binary
+pnpm exec pdpp ref run timeline <run-id>
+```
+
+The public command surface is still the `pdpp` binary; `pnpm exec` is only the
+local workspace launcher.
+
+The local collector runtime is a separate public package:
+
+```bash
+# @pdpp/local-collector package, npx-launched pdpp-local-collector binary
+npx -y @pdpp/local-collector advertise
+
+# @pdpp/local-collector package, installs the pdpp-local-collector binary
+npm i -g @pdpp/local-collector
+pdpp-local-collector advertise
+```
+
 ## Ownership And Publishing
 
 The intended npm scope is `@pdpp`, owned by the durable PDPP/Vana project
@@ -32,6 +75,7 @@ After the package exists on npm, configure the trusted publisher with npm CLI
 11.5.1+:
 
 ```bash
+# npm trust command for the @pdpp/cli package publisher config
 npm trust github @pdpp/cli --repo vana-com/pdpp --file semantic-release.yml
 npm trust list @pdpp/cli
 ```

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@pdpp/cli",
-  "version": "0.1.0-beta.2",
+  "version": "0.1.0-beta.3",
   "description": "Command-line tools for PDPP providers.",
   "type": "module",
   "bin": {
-    "pdpp": "./bin/pdpp.js"
+    "pdpp": "bin/pdpp.js"
   },
   "exports": {
     ".": "./src/index.js",

package/src/cache-layout.js

@@ -5,13 +5,16 @@ export function getPdppCacheLayout(cacheRoot = '.pdpp') {
   return {
     root: cacheRoot,
     clientsDir: join(cacheRoot, 'clients'),
-    grantsDir: join(cacheRoot, 'grants'),
-    secretsDir: join(cacheRoot, 'secrets'),
-    accessFile: join(cacheRoot, 'agent-access.json'),
-    secretFile: (name) => join(cacheRoot, 'secrets', `${name}.secret`),
+    gitignoreFile: join(cacheRoot, '.gitignore'),
+    credentialFile: (providerUrl) => join(cacheRoot, 'clients', `${providerCacheKey(providerUrl)}.json`),
   };
 }
 
+function providerCacheKey(providerUrl) {
+  const host = providerUrl.includes('://') ? new URL(providerUrl).host : providerUrl;
+  return host.replace(/[^a-zA-Z0-9.-]/g, '_');
+}
+
 export function writePdppSecretFile(path, value) {
   mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
   writeFileSync(path, value, { mode: 0o600 });

package/src/collector/commands.js

@@ -0,0 +1,94 @@
+import { PDPP_CLI_BIN_NAME } from '../package-info.js';
+import { CollectorUsageError } from './errors.js';
+import { spawnCollectorRunner } from './runner.js';
+
+const COLLECTOR_HELP = `Local collector runner (reference operator surface).
+
+Pair a host you control with a PDPP reference deployment, then run
+filesystem-class or local-device connectors that the provider container
+cannot run on its own. Runner-owned flags are documented by
+"pdpp-local-collector --help" (see "Distribution" below).
+
+Distribution:
+  The collector runtime ships in @pdpp/local-collector, a separate npm
+  package owned by the PDPP monorepo. @pdpp/cli stays slim and resolves
+  the runner lazily:
+    # @pdpp/local-collector package, installs pdpp-local-collector
+    npm i -g @pdpp/local-collector
+    # @pdpp/local-collector package, npx-launched pdpp-local-collector
+    npx -y @pdpp/local-collector advertise
+  See openspec/changes/publish-pdpp-local-collector/design.md.
+
+Usage:
+  ${PDPP_CLI_BIN_NAME} collector advertise
+  ${PDPP_CLI_BIN_NAME} collector enroll  --base-url <url> --code <one-time-code>
+                          [--device-label <label>]
+  ${PDPP_CLI_BIN_NAME} collector run     --base-url <url> --connector <id>
+                          --device-id <id> --device-token <token>
+                          --connection-id <id>
+                          [--streams a,b,c]
+                          [--backfill-streams attachments]
+                          [--run-id <id>]
+
+Suggested operator flow:
+  1. Start the reference deployment somewhere reachable (e.g. Docker on a
+     server) so it has a base URL such as http://server.local:7662.
+  2. Confirm runtime capabilities with:
+       ${PDPP_CLI_BIN_NAME} collector advertise
+     (@pdpp/cli pdpp shim, resolving @pdpp/local-collector)
+     The collector advertises network, filesystem, local_device
+     and reports collector_protocol_version.
+  3. Mint an enrollment code from the dashboard or "pdpp ref" tooling,
+     then on the host with Claude/Codex data run:
+       ${PDPP_CLI_BIN_NAME} collector enroll --base-url <url> --code <code>
+     (@pdpp/cli pdpp shim, resolving @pdpp/local-collector)
+     The JSON response returns device_id, device_token, and
+     source_instance_id (the connection id for this local binding) —
+     persist all three to a secrets store. You will pass them back as
+     flags/env vars in step 4.
+  4. Run a connector with:
+       PDPP_LOCAL_DEVICE_ID=<device_id> \\
+       PDPP_LOCAL_DEVICE_TOKEN=<device_token> \\
+       PDPP_CONNECTION_ID=<connection_id> \\
+         ${PDPP_CLI_BIN_NAME} collector run --base-url <url> \\
+           --connector claude_code
+     (@pdpp/cli pdpp shim, resolving @pdpp/local-collector)
+     Connectors that need bindings the collector does not advertise fail
+     before spawn with "runtime_capability_mismatch".
+
+Notes:
+  Collector credentials are device-scoped; they cannot read records,
+  approve grants, or mint owner tokens. Do not log device tokens. See
+  openspec/changes/publish-pdpp-local-collector/design.md.
+  Required flags can also be supplied via PDPP_REFERENCE_BASE_URL,
+  PDPP_LOCAL_DEVICE_ID, PDPP_LOCAL_DEVICE_TOKEN, PDPP_CONNECTION_ID,
+  PDPP_RUN_ID. PDPP_SOURCE_INSTANCE_ID remains a compatibility alias.
+`;
+
+const SUBCOMMANDS = new Set(['advertise', 'enroll', 'run']);
+
+export async function runCollector(argv, io) {
+  const [sub, ...rest] = argv;
+
+  if (!sub || sub === '--help' || sub === '-h' || sub === 'help') {
+    io.stdout.write(COLLECTOR_HELP);
+    return 0;
+  }
+
+  if (!SUBCOMMANDS.has(sub)) {
+    io.stderr.write(`Unknown collector subcommand: ${sub}\n\n${COLLECTOR_HELP}`);
+    return 64;
+  }
+
+  try {
+    return await spawnCollectorRunner(sub, rest);
+  } catch (error) {
+    if (error instanceof CollectorUsageError) {
+      io.stderr.write(`${error.message}\n`);
+      return error.exitCode;
+    }
+    throw error;
+  }
+}
+
+export { COLLECTOR_HELP };

package/src/collector/errors.js

@@ -0,0 +1,7 @@
+export class CollectorUsageError extends Error {
+  constructor(message, { exitCode = 64 } = {}) {
+    super(message);
+    this.name = 'CollectorUsageError';
+    this.exitCode = exitCode;
+  }
+}

package/src/collector/runner.js

@@ -0,0 +1,193 @@
+import { spawn } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { CollectorUsageError } from './errors.js';
+
+/**
+ * Resolve the published `@pdpp/local-collector` package, if installed.
+ *
+ * The shim prefers an installed `@pdpp/local-collector` so an operator who
+ * `npm i -g @pdpp/cli && npm i -g @pdpp/local-collector` can run
+ * `pdpp collector ...` without a monorepo checkout. Resolution is lazy —
+ * the CLI does NOT declare a runtime dependency on `@pdpp/local-collector`
+ * (per `publish-pdpp-local-collector` task 4.4); a missing package is
+ * surfaced as an actionable install hint rather than a hard import error.
+ *
+ * Spec: openspec/changes/publish-pdpp-local-collector/design.md §1.
+ */
+export function resolveLocalCollectorPackage(startDir = dirname(fileURLToPath(import.meta.url))) {
+  // Primary resolution: Node module resolution from the caller. Works for an
+  // npm install where @pdpp/local-collector is alongside @pdpp/cli in the
+  // same node_modules tree.
+  try {
+    const require = createRequire(join(startDir, '_'));
+    const manifestPath = require.resolve('@pdpp/local-collector/package.json');
+    return { manifestPath, packageDir: dirname(manifestPath) };
+  } catch {
+    // Continue to workspace fallback.
+  }
+  // Fallback: walk up the directory tree looking for a sibling
+  // packages/local-collector workspace. Preserves the monorepo dev flow
+  // where pnpm does not hoist workspace packages into @pdpp/cli's local
+  // node_modules (per the slim-CLI invariant in task 4.4).
+  let cursor = resolve(startDir);
+  const seen = new Set();
+  while (!seen.has(cursor)) {
+    seen.add(cursor);
+    const candidate = join(cursor, 'packages', 'local-collector', 'package.json');
+    if (existsSync(candidate)) {
+      return { manifestPath: candidate, packageDir: dirname(candidate) };
+    }
+    const parent = dirname(cursor);
+    if (parent === cursor) {
+      break;
+    }
+    cursor = parent;
+  }
+  return null;
+}
+
+/**
+ * Locate the in-monorepo collector-runner TypeScript entrypoint.
+ *
+ * The shim's resolution order is:
+ *
+ *   1. monorepo workspace walk — preserves the current dev flow when
+ *      `pdpp` is invoked from inside a checkout, which uses the
+ *      filesystem-only `bin/collector-runner.ts` directly;
+ *   2. resolved `@pdpp/local-collector` package (via
+ *      `resolveLocalCollectorPackage`);
+ *   3. fail-fast with a one-line install hint.
+ *
+ * This function only handles step 1; the higher-level `spawnCollectorRunner`
+ * weaves the order together so behavior is deterministic across
+ * monorepo + npm install postures.
+ */
+export function resolveCollectorRunnerScript(startDir = dirname(fileURLToPath(import.meta.url))) {
+  let cursor = resolve(startDir);
+  const seen = new Set();
+  while (!seen.has(cursor)) {
+    seen.add(cursor);
+    const candidate = join(cursor, 'packages', 'polyfill-connectors', 'bin', 'collector-runner.ts');
+    if (existsSync(candidate)) {
+      return candidate;
+    }
+    const parent = dirname(cursor);
+    if (parent === cursor) {
+      break;
+    }
+    cursor = parent;
+  }
+  return null;
+}
+
+export function resolveTsxBinary(startDir = dirname(fileURLToPath(import.meta.url))) {
+  let cursor = resolve(startDir);
+  const seen = new Set();
+  while (!seen.has(cursor)) {
+    seen.add(cursor);
+    const candidate = join(cursor, 'node_modules', '.bin', 'tsx');
+    if (existsSync(candidate)) {
+      return candidate;
+    }
+    const parent = dirname(cursor);
+    if (parent === cursor) {
+      break;
+    }
+    cursor = parent;
+  }
+  return null;
+}
+
+/**
+ * One-line install hint surfaced when neither the monorepo nor an
+ * installed `@pdpp/local-collector` can be found.
+ */
+const RUNNER_MISSING_MESSAGE =
+  'pdpp collector requires @pdpp/local-collector. Install once with ' +
+  '"npm i -g @pdpp/local-collector" or run "npx -y @pdpp/local-collector ...". ' +
+  'See openspec/changes/publish-pdpp-local-collector/design.md.';
+
+const TSX_MISSING_MESSAGE =
+  'Could not locate tsx alongside the collector runner. Install ' +
+  '@pdpp/local-collector with "npm i -g @pdpp/local-collector" or run ' +
+  '"pnpm install" at the monorepo root.';
+
+/**
+ * Spawn the collector-runner subprocess. Inherits stdio so operators see
+ * device tokens, run results, and diagnostics directly. Returns the exit
+ * code, never throws on non-zero exits.
+ *
+ * Resolution order, locked in by `publish-pdpp-local-collector` design §1:
+ *   1. monorepo `bin/collector-runner.ts` if walking up the FS finds one;
+ *   2. published `@pdpp/local-collector` bin if installed;
+ *   3. fail-fast `RUNNER_MISSING_MESSAGE`.
+ */
+export async function spawnCollectorRunner(
+  subcommand,
+  argv,
+  {
+    env = process.env,
+    runnerScript = resolveCollectorRunnerScript(),
+    localCollector = resolveLocalCollectorPackage(),
+    tsxBinary = resolveTsxBinary(),
+    spawnFn = spawn,
+    stdio = 'inherit',
+  } = {},
+) {
+  if (runnerScript) {
+    if (!tsxBinary) {
+      throw new CollectorUsageError(TSX_MISSING_MESSAGE);
+    }
+    return await runSubprocess(spawnFn, tsxBinary, [runnerScript, subcommand, ...argv], { env, stdio });
+  }
+
+  if (localCollector) {
+    const binPath = resolveLocalCollectorBin(localCollector.packageDir);
+    if (!existsSync(binPath)) {
+      throw new CollectorUsageError(
+        `@pdpp/local-collector is installed at ${localCollector.packageDir} but is missing its bin entrypoint. ` +
+          'Reinstall the package or report this on https://github.com/vana-com/pdpp/issues.',
+      );
+    }
+    if (binPath.endsWith('.ts')) {
+      if (!tsxBinary) {
+        throw new CollectorUsageError(TSX_MISSING_MESSAGE);
+      }
+      return await runSubprocess(spawnFn, tsxBinary, [binPath, subcommand, ...argv], { env, stdio });
+    }
+    return await runSubprocess(spawnFn, process.execPath, [binPath, subcommand, ...argv], { env, stdio });
+  }
+
+  throw new CollectorUsageError(RUNNER_MISSING_MESSAGE);
+}
+
+function resolveLocalCollectorBin(packageDir) {
+  try {
+    const manifest = JSON.parse(readFileSync(join(packageDir, 'package.json'), 'utf8'));
+    const bin = manifest?.bin?.['pdpp-local-collector'];
+    if (typeof bin === 'string' && bin.trim()) {
+      return join(packageDir, bin);
+    }
+  } catch {}
+  const publishedBin = join(packageDir, 'dist', 'local-collector', 'bin', 'pdpp-local-collector.js');
+  if (existsSync(publishedBin)) return publishedBin;
+  return join(packageDir, 'bin', 'pdpp-local-collector.ts');
+}
+
+function runSubprocess(spawnFn, binary, args, { env, stdio }) {
+  return new Promise((resolvePromise, rejectPromise) => {
+    const child = spawnFn(binary, args, { env, stdio });
+    child.on('error', rejectPromise);
+    child.on('exit', (code, signal) => {
+      if (signal) {
+        rejectPromise(new Error(`collector-runner terminated by signal ${signal}`));
+        return;
+      }
+      resolvePromise(code ?? 0);
+    });
+  });
+}

package/src/connect/flow.js

@@ -53,11 +53,24 @@ export async function connectProvider(providerUrl, options = {}) {
     );
   }
 
-  const start = await postJson(fetchFn, connectEndpoint, {
+  const publicClient = await getOrRegisterPublicClient({
+    fetchFn,
+    authorizationMetadata,
+    cacheRoot,
+    providerUrl: normalizedProviderUrl,
+    clientName: 'PDPP CLI',
+  });
+
+  const startRequest = {
     resource: normalizedProviderUrl,
     scope,
     client_name: 'PDPP CLI',
-  });
+  };
+  if (publicClient?.client_id) {
+    startRequest.client_id = publicClient.client_id;
+  }
+
+  const start = await postJson(fetchFn, connectEndpoint, startRequest);
 
   const approvalUrl = start.approval_url ?? start.verification_uri_complete ?? start.verification_uri;
   const pollUrl = start.poll_url ?? start.token_url ?? start.device_poll_endpoint ?? start.completion_endpoint;
@@ -87,6 +100,7 @@ export async function connectProvider(providerUrl, options = {}) {
     provider_url: normalizedProviderUrl,
     authorization_server: authorizationServerUrl,
     scope,
+    client: publicClient,
     credential,
     created_at: new Date((options.now?.() ?? Date.now())).toISOString(),
   });
@@ -100,9 +114,50 @@ export async function connectProvider(providerUrl, options = {}) {
     authorizationServerUrl,
     cacheFile,
     scope,
+    clientId: publicClient?.client_id ?? null,
   };
 }
 
+export async function readStoredCredential(providerUrl, options = {}) {
+  const normalizedProviderUrl = normalizeProviderUrl(providerUrl);
+  if (!normalizedProviderUrl) {
+    throw new ConnectError('invalid_provider_url', `Invalid provider URL: ${providerUrl}`, 64);
+  }
+
+  const cacheRoot = options.cacheRoot ?? '.pdpp';
+  const cacheFile = getCredentialCacheFile(cacheRoot, normalizedProviderUrl);
+  let payload;
+  try {
+    payload = JSON.parse(await readFile(cacheFile, 'utf8'));
+  } catch (error) {
+    if (error?.code === 'ENOENT') {
+      throw new ConnectError(
+        'not_connected',
+        `No PDPP credential found for ${normalizedProviderUrl}. Run pdpp connect ${normalizedProviderUrl} first.`
+      );
+    }
+    throw error;
+  }
+
+  const credential = payload?.credential;
+  if (!credential?.access_token) {
+    throw new ConnectError('credential_invalid', `Credential cache entry is missing an access token: ${cacheFile}`);
+  }
+
+  if (credential.expires_at) {
+    const expiresAtMs = Date.parse(credential.expires_at);
+    const now = options.now?.() ?? Date.now();
+    if (Number.isFinite(expiresAtMs) && expiresAtMs <= now) {
+      throw new ConnectError(
+        'credential_expired',
+        `Credential for ${normalizedProviderUrl} expired. Run pdpp connect ${normalizedProviderUrl} again.`
+      );
+    }
+  }
+
+  return { cacheFile, payload, credential, providerUrl: normalizedProviderUrl };
+}
+
 export function normalizeProviderUrl(value) {
   try {
     const parsed = new URL(value.includes('://') ? value : `https://${value}`);
@@ -167,6 +222,56 @@ function findAgentConnectEndpoint(resourceMetadata, authorizationMetadata) {
   }
 }
 
+function findRegistrationEndpoint(authorizationMetadata) {
+  const endpoint = authorizationMetadata.registration_endpoint;
+  if (!endpoint) {
+    return null;
+  }
+  const modes = authorizationMetadata.pdpp_registration_modes_supported;
+  if (Array.isArray(modes) && !modes.includes('dynamic')) {
+    return null;
+  }
+  try {
+    return new URL(endpoint, authorizationMetadata.issuer).toString();
+  } catch {
+    throw new ConnectError('metadata_failure', 'Registration endpoint in provider metadata is not a valid URL.');
+  }
+}
+
+async function getOrRegisterPublicClient({ fetchFn, authorizationMetadata, cacheRoot, providerUrl, clientName }) {
+  const cached = await readCachedClientRegistration(cacheRoot, providerUrl);
+  if (cached) {
+    return cached;
+  }
+  const registrationEndpoint = findRegistrationEndpoint(authorizationMetadata);
+  if (!registrationEndpoint) {
+    return null;
+  }
+  const registered = await postJson(fetchFn, registrationEndpoint, {
+    client_name: clientName,
+    token_endpoint_auth_method: 'none',
+  });
+  if (!registered?.client_id) {
+    throw new ConnectError('connect_contract_invalid', 'Dynamic client registration response did not include client_id.');
+  }
+  return {
+    client_id: registered.client_id,
+    client_name: registered.client_name ?? clientName,
+    token_endpoint_auth_method: registered.token_endpoint_auth_method ?? 'none',
+  };
+}
+
+async function readCachedClientRegistration(cacheRoot, providerUrl) {
+  try {
+    const payload = JSON.parse(await readFile(getCredentialCacheFile(cacheRoot, providerUrl), 'utf8'));
+    const client = payload?.client;
+    return client?.client_id ? client : null;
+  } catch (error) {
+    if (error?.code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
 async function pollForCredential(fetchFn, pollUrl, options) {
   const startedAt = options.now?.() ?? Date.now();
   const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
@@ -194,6 +299,7 @@ async function pollForCredential(fetchFn, pollUrl, options) {
         access_token: credential.access_token,
         token_type: credential.token_type ?? 'Bearer',
         expires_at: credential.expires_at,
+        grant_id: credential.grant_id ?? result.grant_id,
         scope: credential.scope,
       };
     }
@@ -238,14 +344,18 @@ async function verifySchema(fetchFn, providerUrl, accessToken) {
 }
 
 async function storeCredential(cacheRoot, providerUrl, payload) {
-  const host = new URL(providerUrl).host.replace(/[^a-zA-Z0-9.-]/g, '_');
-  const cacheFile = join(cacheRoot, 'clients', `${host}.json`);
+  const cacheFile = getCredentialCacheFile(cacheRoot, providerUrl);
   await mkdir(dirname(cacheFile), { recursive: true, mode: 0o700 });
   await ensurePdppGitignore(cacheRoot);
   await writeFile(cacheFile, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 });
   return cacheFile;
 }
 
+function getCredentialCacheFile(cacheRoot, providerUrl) {
+  const host = new URL(providerUrl).host.replace(/[^a-zA-Z0-9.-]/g, '_');
+  return join(cacheRoot, 'clients', `${host}.json`);
+}
+
 async function ensurePdppGitignore(cacheRoot) {
   const gitignorePath = join(cacheRoot, '.gitignore');
   let current = '';