palaryn 0.1.0 → 0.3.2

package/src/server/stream-proxy.ts

@@ -0,0 +1,202 @@
+import * as http from 'http';
+import * as https from 'https';
+import { URL } from 'url';
+import type { Response as ExpressResponse } from 'express';
+import { isPrivateIP } from '../executor/http-executor';
+import * as dns from 'dns';
+import * as net from 'net';
+
+/** Maximum accumulated response size for post-DLP scan (50 MB) */
+const MAX_ACCUMULATED_BYTES = 50 * 1024 * 1024;
+
+/**
+ * Resolve a hostname and verify it's not private/internal (SSRF protection).
+ * Reuses the same logic as HttpExecutor.
+ */
+function validateResolvedIP(hostname: string): Promise<void> {
+  if (net.isIP(hostname)) {
+    if (isPrivateIP(hostname)) {
+      return Promise.reject(new Error(`SSRF blocked: resolved IP ${hostname} is a private/reserved address`));
+    }
+    return Promise.resolve();
+  }
+
+  return new Promise((resolve, reject) => {
+    dns.lookup(hostname, { all: true }, (err, addresses) => {
+      if (err) return reject(err);
+      for (const addr of addresses) {
+        if (isPrivateIP(addr.address)) {
+          return reject(
+            new Error(`SSRF blocked: hostname "${hostname}" resolved to private IP ${addr.address}`)
+          );
+        }
+      }
+      resolve();
+    });
+  });
+}
+
+export interface StreamProxyOptions {
+  /** Target URL to proxy to */
+  url: string;
+  /** HTTP method (default: POST) */
+  method?: string;
+  /** Request headers to forward */
+  headers?: Record<string, string>;
+  /** Request body */
+  body?: unknown;
+  /** Timeout in milliseconds (default: 120000) */
+  timeoutMs?: number;
+  /** Whether to enable SSRF protection (default: true) */
+  ssrfProtection?: boolean;
+}
+
+export interface StreamProxyResult {
+  /** Accumulated response body (all SSE chunks concatenated) */
+  accumulatedBody: string;
+  /** Upstream HTTP status code */
+  httpStatus: number;
+  /** Upstream response headers */
+  headers: Record<string, string>;
+}
+
+/**
+ * Opens an HTTP(S) connection to the target URL and pipes the response
+ * as SSE events to an Express response. Accumulates the full response
+ * in a buffer for post-stream DLP scanning.
+ *
+ * @returns The accumulated response body and upstream metadata on stream end.
+ */
+export function streamProxy(
+  options: StreamProxyOptions,
+  clientRes: ExpressResponse,
+): Promise<StreamProxyResult> {
+  const {
+    url,
+    method = 'POST',
+    headers = {},
+    body,
+    timeoutMs = 120000,
+    ssrfProtection = true,
+  } = options;
+
+  return new Promise(async (resolve, reject) => {
+    // SSRF protection
+    const parsedUrl = new URL(url);
+    if (ssrfProtection) {
+      try {
+        await validateResolvedIP(parsedUrl.hostname);
+      } catch (err) {
+        reject(err);
+        return;
+      }
+    }
+
+    const isHttps = parsedUrl.protocol === 'https:';
+    const transport = isHttps ? https : http;
+    const bodyStr = body ? (typeof body === 'string' ? body : JSON.stringify(body)) : undefined;
+
+    const requestHeaders: Record<string, string> = { ...headers };
+    if (bodyStr && !requestHeaders['content-type'] && !requestHeaders['Content-Type']) {
+      requestHeaders['Content-Type'] = 'application/json';
+    }
+    if (bodyStr) {
+      requestHeaders['Content-Length'] = Buffer.byteLength(bodyStr).toString();
+    }
+
+    const requestOptions = {
+      hostname: parsedUrl.hostname,
+      port: parsedUrl.port || (isHttps ? 443 : 80),
+      path: parsedUrl.pathname + parsedUrl.search,
+      method: method.toUpperCase(),
+      headers: requestHeaders,
+      timeout: timeoutMs,
+    };
+
+    const req = transport.request(requestOptions, (upstreamRes) => {
+      const httpStatus = upstreamRes.statusCode || 502;
+
+      // Collect upstream response headers
+      const responseHeaders: Record<string, string> = {};
+      for (const [key, value] of Object.entries(upstreamRes.headers)) {
+        if (value) responseHeaders[key] = Array.isArray(value) ? value.join(', ') : value;
+      }
+
+      // If upstream returns non-2xx, read the body and reject
+      if (httpStatus >= 400) {
+        let errBody = '';
+        upstreamRes.on('data', (chunk: Buffer) => { errBody += chunk.toString(); });
+        upstreamRes.on('end', () => {
+          reject(new Error(`Upstream returned HTTP ${httpStatus}: ${errBody.slice(0, 500)}`));
+        });
+        return;
+      }
+
+      // Set SSE response headers on the client response
+      clientRes.writeHead(200, {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'Connection': 'keep-alive',
+        'X-Accel-Buffering': 'no',
+      });
+
+      // Accumulation buffer for post-DLP
+      let accumulated = '';
+      let accumulatedBytes = 0;
+
+      upstreamRes.on('data', (chunk: Buffer) => {
+        const str = chunk.toString();
+
+        // Pipe directly to client (zero-buffer streaming)
+        clientRes.write(chunk);
+
+        // Accumulate for post-stream DLP scan
+        accumulatedBytes += chunk.length;
+        if (accumulatedBytes <= MAX_ACCUMULATED_BYTES) {
+          accumulated += str;
+        }
+      });
+
+      upstreamRes.on('end', () => {
+        clientRes.end();
+        resolve({
+          accumulatedBody: accumulated,
+          httpStatus,
+          headers: responseHeaders,
+        });
+      });
+
+      upstreamRes.on('error', (err) => {
+        // Send SSE error event to client before closing
+        try {
+          clientRes.write(`event: error\ndata: ${JSON.stringify({ error: err.message })}\n\n`);
+          clientRes.end();
+        } catch {
+          // Client may already be disconnected
+        }
+        reject(err);
+      });
+    });
+
+    req.on('error', (err) => {
+      reject(err);
+    });
+
+    req.on('timeout', () => {
+      req.destroy();
+      reject(new Error(`Upstream request timeout after ${timeoutMs}ms`));
+    });
+
+    // Handle client disconnect — abort the upstream request
+    clientRes.on('close', () => {
+      if (!req.destroyed) {
+        req.destroy();
+      }
+    });
+
+    if (bodyStr) {
+      req.write(bodyStr);
+    }
+    req.end();
+  });
+}

package/src/storage/file-persistence.ts

@@ -0,0 +1,315 @@
+/**
+ * File-based persistence for SaaS in-memory stores.
+ * Saves all store data to a JSON file on every write operation.
+ * Loads data from disk on startup.
+ *
+ * This is for development only -- production uses PostgreSQL.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import {
+  User,
+  OAuthAccount,
+  OAuthProvider,
+  Workspace,
+  WorkspaceMember,
+  Session,
+  UserApiKey,
+} from '../types/user';
+import { PolicyPack } from '../types/policy';
+import {
+  UserStore,
+  OAuthAccountStore,
+  WorkspaceStore,
+  WorkspaceMemberStore,
+  SessionStore,
+  UserApiKeyStore,
+  PolicyStore,
+} from './interfaces';
+import {
+  InMemoryUserStore,
+  InMemoryOAuthAccountStore,
+  InMemoryWorkspaceStore,
+  InMemoryWorkspaceMemberStore,
+  InMemorySessionStore,
+  InMemoryUserApiKeyStore,
+  InMemoryPolicyStore,
+} from './memory';
+import { OAuthClientsStore } from '../mcp/oauth-stores';
+import { OAuthClientInformationFull } from '@modelcontextprotocol/sdk/shared/auth.js';
+
+interface PersistedData {
+  users: [string, User][];
+  oauthAccounts: [string, OAuthAccount][];
+  workspaces: [string, Workspace][];
+  members: [string, WorkspaceMember][];
+  sessions: [string, Session][];
+  apiKeys: [string, UserApiKey][];
+  policies: [string, PolicyPack][];
+  /** MCP OAuth clients (dynamic registration) */
+  mcpOAuthClients?: [string, OAuthClientInformationFull][];
+}
+
+/**
+ * Wraps an in-memory store with file persistence.
+ * Every mutation triggers a save of ALL stores to disk.
+ */
+export class FilePersistedStores {
+  private filePath: string;
+  private saveTimer: ReturnType<typeof setTimeout> | null = null;
+
+  readonly userStore: UserStore;
+  readonly oauthAccountStore: OAuthAccountStore;
+  readonly workspaceStore: WorkspaceStore;
+  readonly workspaceMemberStore: WorkspaceMemberStore;
+  readonly sessionStore: SessionStore;
+  readonly userApiKeyStore: UserApiKeyStore;
+  readonly policyStore: PolicyStore;
+  readonly oauthClientsStore: OAuthClientsStore;
+
+  // Keep references to the underlying in-memory stores for serialization
+  private _users: InMemoryUserStore;
+  private _oauth: InMemoryOAuthAccountStore;
+  private _workspaces: InMemoryWorkspaceStore;
+  private _members: InMemoryWorkspaceMemberStore;
+  private _sessions: InMemorySessionStore;
+  private _apiKeys: InMemoryUserApiKeyStore;
+  private _policies: InMemoryPolicyStore;
+  private _oauthClients: OAuthClientsStore;
+
+  constructor(dataDir: string) {
+    fs.mkdirSync(dataDir, { recursive: true });
+    this.filePath = path.join(dataDir, 'saas-state.json');
+
+    this._users = new InMemoryUserStore();
+    this._oauth = new InMemoryOAuthAccountStore();
+    this._workspaces = new InMemoryWorkspaceStore();
+    this._members = new InMemoryWorkspaceMemberStore();
+    this._sessions = new InMemorySessionStore();
+    this._apiKeys = new InMemoryUserApiKeyStore();
+    this._policies = new InMemoryPolicyStore();
+    this._oauthClients = new OAuthClientsStore();
+
+    // Load existing data from disk
+    this.load();
+
+    // Create proxied stores that auto-save on mutations
+    const save = () => this.scheduleSave();
+    const saveNow = () => this.scheduleSave(true);
+
+    this.userStore = this.wrapUserStore(this._users, save, saveNow);
+    this.oauthAccountStore = this.wrapOAuthStore(this._oauth, save);
+    this.workspaceStore = this.wrapWorkspaceStore(this._workspaces, save);
+    this.workspaceMemberStore = this.wrapMemberStore(this._members, save);
+    this.sessionStore = this.wrapSessionStore(this._sessions, save);
+    this.userApiKeyStore = this.wrapApiKeyStore(this._apiKeys, save, saveNow);
+    this.policyStore = this.wrapPolicyStore(this._policies, save);
+    this.oauthClientsStore = this._oauthClients; // Direct reference; persisted via saveNow on load
+  }
+
+  private load(): void {
+    try {
+      if (!fs.existsSync(this.filePath)) return;
+      const raw = fs.readFileSync(this.filePath, 'utf-8');
+      const data: PersistedData = JSON.parse(raw);
+
+      for (const [, u] of data.users || []) this._users.create(u);
+      for (const [, a] of data.oauthAccounts || []) this._oauth.create(a);
+      for (const [, w] of data.workspaces || []) this._workspaces.create(w);
+      for (const [, m] of data.members || []) this._members.create(m);
+      for (const [, s] of data.sessions || []) this._sessions.create(s);
+      for (const [, k] of data.apiKeys || []) this._apiKeys.create(k);
+      for (const [wsId, p] of data.policies || []) this._policies.set(wsId, p);
+      if (data.mcpOAuthClients) {
+        this._oauthClients.load(data.mcpOAuthClients);
+      }
+
+      const counts = [
+        data.users?.length || 0,
+        data.workspaces?.length || 0,
+        data.apiKeys?.length || 0,
+      ];
+      console.log(`Loaded persisted state: ${counts[0]} users, ${counts[1]} workspaces, ${counts[2]} API keys`);
+    } catch (err) {
+      console.warn('Failed to load persisted state:', (err as Error).message);
+    }
+  }
+
+  private scheduleSave(immediate = false): void {
+    if (immediate) {
+      // Cancel any pending debounced save and write synchronously now
+      if (this.saveTimer) {
+        clearTimeout(this.saveTimer);
+        this.saveTimer = null;
+      }
+      this.saveNow();
+      return;
+    }
+    // Debounce saves to avoid excessive disk writes
+    if (this.saveTimer) return;
+    this.saveTimer = setTimeout(() => {
+      this.saveTimer = null;
+      this.saveNow();
+    }, 100);
+  }
+
+  /**
+   * Write to disk synchronously. Use for critical operations where
+   * data loss from a crash within 100ms would be unacceptable
+   * (e.g. user creation, API key creation).
+   */
+  saveImmediate(): void {
+    if (this.saveTimer) {
+      clearTimeout(this.saveTimer);
+      this.saveTimer = null;
+    }
+    this.saveNow();
+  }
+
+  private saveNow(): void {
+    try {
+      const data: PersistedData = {
+        users: this._users.list().map(u => [u.id, u]),
+        oauthAccounts: [], // OAuth accounts are ephemeral
+        workspaces: this._workspaces.list().map(w => [w.id, w]),
+        members: this.serializeMembers(),
+        sessions: [], // Sessions are ephemeral
+        apiKeys: this.serializeApiKeys(),
+        policies: this.serializePolicies(),
+        mcpOAuthClients: this._oauthClients.entries(),
+      };
+      const tmp = this.filePath + '.tmp';
+      fs.writeFileSync(tmp, JSON.stringify(data, null, 2));
+      fs.renameSync(tmp, this.filePath);
+    } catch (err) {
+      console.error('Failed to persist state:', (err as Error).message);
+    }
+  }
+
+  private serializeMembers(): [string, WorkspaceMember][] {
+    // Access members via store's list-like queries (workspace-based)
+    const workspaces = this._workspaces.list();
+    const seen = new Set<string>();
+    const result: [string, WorkspaceMember][] = [];
+    for (const w of workspaces) {
+      for (const m of this._members.getByWorkspace(w.id)) {
+        if (!seen.has(m.id)) {
+          seen.add(m.id);
+          result.push([m.id, m]);
+        }
+      }
+    }
+    return result;
+  }
+
+  private serializeApiKeys(): [string, UserApiKey][] {
+    const workspaces = this._workspaces.list();
+    const seen = new Set<string>();
+    const result: [string, UserApiKey][] = [];
+    for (const w of workspaces) {
+      for (const k of this._apiKeys.getByWorkspace(w.id)) {
+        if (!seen.has(k.id)) {
+          seen.add(k.id);
+          result.push([k.id, k]);
+        }
+      }
+    }
+    return result;
+  }
+
+  private serializePolicies(): [string, PolicyPack][] {
+    const m = this._policies.list();
+    return Array.from(m.entries());
+  }
+
+  flush(): void {
+    if (this.saveTimer) {
+      clearTimeout(this.saveTimer);
+      this.saveTimer = null;
+    }
+    this.saveNow();
+  }
+
+  // --- Wrapper factories that add save-on-mutate ---
+
+  private wrapUserStore(inner: InMemoryUserStore, save: () => void, saveNow: () => void): UserStore {
+    return {
+      create(user: User) { inner.create(user); saveNow(); },
+      getById(id: string) { return inner.getById(id); },
+      getByEmail(email: string) { return inner.getByEmail(email); },
+      update(id: string, updates: any) { const r = inner.update(id, updates); save(); return r; },
+      delete(id: string) { const r = inner.delete(id); save(); return r; },
+      list() { return inner.list(); },
+    };
+  }
+
+  private wrapOAuthStore(inner: InMemoryOAuthAccountStore, save: () => void): OAuthAccountStore {
+    return {
+      create(account: OAuthAccount) { inner.create(account); save(); },
+      getById(id: string) { return inner.getById(id); },
+      getByProvider(provider: OAuthProvider, providerUserId: string) { return inner.getByProvider(provider, providerUserId); },
+      getByUserId(userId: string) { return inner.getByUserId(userId); },
+      update(id: string, updates: any) { const r = inner.update(id, updates); save(); return r; },
+      delete(id: string) { const r = inner.delete(id); save(); return r; },
+    };
+  }
+
+  private wrapWorkspaceStore(inner: InMemoryWorkspaceStore, save: () => void): WorkspaceStore {
+    return {
+      create(workspace: Workspace) { inner.create(workspace); save(); },
+      getById(id: string) { return inner.getById(id); },
+      getBySlug(slug: string) { return inner.getBySlug(slug); },
+      getByOwner(userId: string) { return inner.getByOwner(userId); },
+      update(id: string, updates: any) { const r = inner.update(id, updates); save(); return r; },
+      delete(id: string) { const r = inner.delete(id); save(); return r; },
+      list() { return inner.list(); },
+    };
+  }
+
+  private wrapMemberStore(inner: InMemoryWorkspaceMemberStore, save: () => void): WorkspaceMemberStore {
+    return {
+      create(member: WorkspaceMember) { inner.create(member); save(); },
+      getById(id: string) { return inner.getById(id); },
+      getByWorkspace(workspaceId: string) { return inner.getByWorkspace(workspaceId); },
+      getByUser(userId: string) { return inner.getByUser(userId); },
+      getByWorkspaceAndUser(workspaceId: string, userId: string) { return inner.getByWorkspaceAndUser(workspaceId, userId); },
+      update(id: string, updates: any) { const r = inner.update(id, updates); save(); return r; },
+      delete(id: string) { const r = inner.delete(id); save(); return r; },
+    };
+  }
+
+  private wrapSessionStore(inner: InMemorySessionStore, save: () => void): SessionStore {
+    return {
+      create(session: Session) { inner.create(session); save(); },
+      getById(id: string) { return inner.getById(id); },
+      getByUserId(userId: string) { return inner.getByUserId(userId); },
+      update(id: string, updates: any) { const r = inner.update(id, updates); save(); return r; },
+      delete(id: string) { const r = inner.delete(id); save(); return r; },
+      deleteExpired() { const r = inner.deleteExpired(); save(); return r; },
+      deleteByUserId(userId: string) { const r = inner.deleteByUserId(userId); save(); return r; },
+    };
+  }
+
+  private wrapApiKeyStore(inner: InMemoryUserApiKeyStore, save: () => void, saveNow: () => void): UserApiKeyStore {
+    return {
+      create(apiKey: UserApiKey) { inner.create(apiKey); saveNow(); },
+      getById(id: string) { return inner.getById(id); },
+      getByKeyHash(keyHash: string) { return inner.getByKeyHash(keyHash); },
+      verifyToken(token: string) { return inner.verifyToken(token); },
+      getByWorkspace(workspaceId: string) { return inner.getByWorkspace(workspaceId); },
+      getByUser(userId: string) { return inner.getByUser(userId); },
+      update(id: string, updates: any) { const r = inner.update(id, updates); save(); return r; },
+      delete(id: string) { const r = inner.delete(id); save(); return r; },
+    };
+  }
+
+  private wrapPolicyStore(inner: InMemoryPolicyStore, save: () => void): PolicyStore {
+    return {
+      getByWorkspaceId(workspaceId: string) { return inner.getByWorkspaceId(workspaceId); },
+      set(workspaceId: string, policy: PolicyPack) { inner.set(workspaceId, policy); save(); },
+      delete(workspaceId: string) { const r = inner.delete(workspaceId); save(); return r; },
+      list() { return inner.list(); },
+      clear() { inner.clear(); save(); },
+    };
+  }
+}

package/src/storage/index.ts

@@ -0,0 +1,4 @@
+export * from './interfaces';
+export * from './memory';
+export * from './postgres';
+export * from './redis';