newo 1.4.0 → 1.5.1

package/.env.example

@@ -1,14 +1,25 @@
 # NEWO endpoints
 NEWO_BASE_URL=https://app.newo.ai
 
-# Project you want to sync (optional - leave blank to sync all accessible projects)
-# NEWO_PROJECT_ID=b78188ba-0df0-46a8-8713-f0d7cff0a06e
+# Multi-customer configuration - Array format
+# Define API keys as JSON array - customer IDN will be auto-detected from API response
+NEWO_API_KEYS=["api_key_1", "api_key_2", "api_key_3"]
+
+# Or with optional project IDs per API key
+# NEWO_API_KEYS=[{"key":"api_key_1","project_id":"project_uuid_1"}, {"key":"api_key_2","project_id":"project_uuid_2"}]
+
+# Optional: specify default customer IDN after keys are loaded
+# NEWO_DEFAULT_CUSTOMER=NEWO_ESd2BC95
 
-# Auth (choose one)
-# 1) Recommended: API key that can be exchanged for tokens:
-NEWO_API_KEY=put_api_key_here
+# Legacy format (still supported)
+# NEWO_CUSTOMER_[IDN]_API_KEY=api_key
+# NEWO_CUSTOMER_acme_API_KEY=put_acme_api_key_here
+
+# Legacy single-customer mode (still supported)
+# NEWO_API_KEY=put_api_key_here
+# NEWO_PROJECT_ID=b78188ba-0df0-46a8-8713-f0d7cff0a06e
 
-# 2) Optional bootstrap tokens (used if present)
+# Optional bootstrap tokens (for legacy mode or manual setup)
 NEWO_ACCESS_TOKEN=
 NEWO_REFRESH_TOKEN=
 

package/CHANGELOG.md

@@ -5,6 +5,108 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.5.1] - 2025-01-14
+
+### Added
+- **Comprehensive Test Coverage**: Added extensive test suites for all major modules
+  - `test/auth.test.js`: 500+ lines covering authentication, token management, multi-customer support
+  - `test/hash.test.js`: 400+ lines covering SHA256 hashing, hash storage, and cross-platform compatibility  
+  - `test/fsutil.test.js`: 400+ lines covering file system utilities and path handling
+  - `test/akb.test.js`: 600+ lines covering AKB article parsing and import workflows
+  - Added missing test dependencies: `chai`, `sinon`, `c8` for coverage reporting
+- **Enhanced Authentication Validation**:
+  - `validateApiKey()`: Comprehensive API key format and length validation
+  - `validateTokens()`: Token format and structure validation with detailed error messages
+  - `validateUrl()`: URL format validation for API endpoints
+  - Sensitive data sanitization in logs (API keys and tokens masked)
+- **Structured Logging System**:
+  - `logAuthEvent()`: Structured authentication event logging with metadata
+  - Automatic sensitive data sanitization (keys/tokens/secrets masked in logs)
+  - JSON-formatted logs with timestamp, level, module, and context information
+- **Enhanced Error Handling**:
+  - User-friendly CLI error messages with troubleshooting tips
+  - Specific error handling for authentication, network, environment, and file system errors
+  - Verbose mode support for detailed debugging information
+  - Context-aware error messages with suggested solutions
+
+### Enhanced
+- **Authentication Robustness** (`src/auth.ts`):
+  - Added comprehensive input validation with detailed error messages
+  - Enhanced network error handling with specific status code interpretation
+  - Added request timeouts (30 seconds) and retry logic for reliability
+  - Improved token expiry handling with 60-second buffer for refresh
+  - Better handling of connection errors, timeouts, and server errors
+- **CLI Error Experience** (`src/cli.ts`):
+  - Added `handleCliError()` function with categorized error types
+  - User-friendly error messages with emoji indicators and troubleshooting tips
+  - Verbose mode toggle for detailed technical information vs. clean user messages
+  - Specific guidance for common issues (API key, network, configuration)
+- **Testing Infrastructure**:
+  - Fixed ES module/CommonJS compatibility issues in test files
+  - Enhanced `TestEnvironment` class with comprehensive cleanup and mocking
+  - Added MockHttpClient, MockFileSystem, and MockLogger utilities
+  - Comprehensive assertion helpers and test data generators
+
+### Fixed
+- **Module System Compatibility**: Resolved ES module/CommonJS conflicts in test environment
+- **Test Dependencies**: Added missing testing dependencies that were imported but not declared
+- **Integration Test Paths**: Fixed paths from `src/cli.js` to `dist/cli.js` for proper compiled code testing
+- **Error Message Consistency**: Standardized error messages across authentication and CLI modules
+
+### Technical Details
+- **Validation Constants**: Added security-focused validation thresholds (API_KEY_MIN_LENGTH, TOKEN_MIN_LENGTH)
+- **Request Configuration**: Added proper timeout handling (30s) and user-agent headers
+- **Error Recovery**: Comprehensive fallback strategies for different failure scenarios
+- **Logging Standards**: JSON-structured logs with automatic PII/sensitive data protection
+- **Test Coverage**: Achieved comprehensive test coverage across all core modules with realistic scenarios
+
+### Developer Experience
+- **Enhanced Debugging**: Verbose mode provides detailed technical information for troubleshooting
+- **Better Error Messages**: Clear, actionable error messages instead of generic API errors
+- **Comprehensive Testing**: Full test suite covering authentication, file operations, hashing, and AKB import
+- **Type Safety**: All improvements maintain strict TypeScript compliance with proper error types
+
+## [1.5.0] - 2025-09-03
+
+### Changed
+- **Complete TypeScript Refactoring**: Major codebase conversion from JavaScript to TypeScript
+  - All source files converted to TypeScript with `.ts` extensions
+  - Added comprehensive type definitions in `src/types.ts`
+  - Strict TypeScript configuration with `exactOptionalPropertyTypes` and `noUncheckedIndexedAccess`
+  - Modern ES2022 target with ESNext modules for optimal performance
+  - Enhanced IntelliSense support and developer experience
+
+### Added
+- **TypeScript Build System**: 
+  - `tsconfig.json` with strict type checking and modern ES features
+  - New build scripts: `npm run build`, `npm run build:watch`, `npm run typecheck`
+  - Development scripts: `npm run dev`, `npm run pull`, `npm run push`, `npm run status`
+  - Source map generation for debugging compiled JavaScript
+- **Enhanced Type Safety**:
+  - Complete type definitions for all NEWO API responses and data structures
+  - Strict error handling with proper TypeScript error types
+  - Optional property handling with explicit `| undefined` types
+  - Enhanced Axios integration with proper TypeScript interceptor types
+
+### Technical Details
+- **Type Definitions**: Comprehensive interfaces for `ProjectMeta`, `Agent`, `Flow`, `Skill`, `FlowEvent`, `FlowState`, and all API responses
+- **Build Output**: TypeScript compiles to `dist/` directory with JavaScript and declaration files
+- **Import Strategy**: Uses runtime `.js` extensions in TypeScript source (required for ESModules)
+- **Dependency Updates**: Added TypeScript and @types packages for full type support
+- **Package.json**: Updated with TypeScript build pipeline and development scripts
+
+### Migration for Developers
+- **New Development Workflow**: `npm run build` required before running CLI commands
+- **Source Files**: All development now in `src/*.ts` files instead of `src/*.js`
+- **Build Artifacts**: Generated JavaScript in `dist/` directory (automatically created)
+- **IDE Support**: Enhanced autocomplete, error detection, and refactoring capabilities
+
+### Backward Compatibility
+- **Runtime Behavior**: No changes to CLI command interface or functionality
+- **Environment Variables**: All existing `.env` configurations continue to work
+- **File Formats**: Same `.guidance` and `.jinja` file support as before
+- **API Compatibility**: No changes to NEWO API integration or endpoints
+
 ## [1.4.0] - 2025-08-20
 
 ### Added

package/README.md

@@ -33,6 +33,7 @@ npx newo status
 git clone https://github.com/sabbah13/newo-cli.git
 cd newo-cli
 npm install
+npm run build    # Build TypeScript to JavaScript
 ```
 
 ## Configure
@@ -92,8 +93,40 @@ Hashes are tracked in `.newo/hashes.json` so only changed files are pushed.
 - **AKB import**: Import knowledge base articles from structured text files
 - **Project structure export**: Generates `flows.yaml` with complete project metadata
 - **Robust authentication**: API key exchange with automatic token refresh
+- **Enhanced error handling**: User-friendly error messages with troubleshooting guidance
+- **Comprehensive testing**: Full test suite covering all major functionality
 - **CI/CD ready**: GitHub Actions workflow included
 
+## Robustness & Error Handling
+
+NEWO CLI v1.5.1+ includes comprehensive error handling and validation:
+
+### User-Friendly Error Messages
+- **Authentication Errors**: Clear guidance when API keys are invalid or missing
+- **Network Issues**: Helpful tips for connection problems and timeouts  
+- **Configuration Errors**: Step-by-step setup instructions for common issues
+- **File System Errors**: Actionable guidance for permission and path problems
+
+### Verbose Debugging
+Use the `--verbose` or `-v` flag with any command for detailed technical information:
+```bash
+npx newo pull --verbose     # Detailed pull operation logs
+npx newo push -v           # Verbose push with full error context
+```
+
+### Enhanced Validation
+- **API Key Validation**: Format and length validation with specific error messages
+- **Token Security**: Automatic sanitization of sensitive data in logs
+- **Network Timeouts**: 30-second request timeouts with proper error handling
+- **Input Validation**: Comprehensive validation for all user inputs and configuration
+
+### Troubleshooting Tips
+When errors occur, NEWO CLI provides:
+- 🔍 **Problem diagnosis** with specific error categories
+- 💡 **Solution suggestions** for common configuration issues
+- 📋 **Step-by-step guidance** for resolving authentication and network problems
+- 🔧 **Configuration validation** to ensure proper setup
+
 ## CI/CD (GitHub Actions)
 Create `.github/workflows/deploy.yml`:
 ```yaml
@@ -113,7 +146,7 @@ jobs:
         with:
           node-version: 20
       - run: npm ci
-      - run: node ./src/cli.js push
+      - run: npm run build && node ./dist/cli.js push
         env:
           NEWO_BASE_URL: https://app.newo.ai
           NEWO_PROJECT_ID: ${{ secrets.NEWO_PROJECT_ID }}
@@ -155,6 +188,68 @@ Each article will be imported with:
 
 Use `--verbose` flag to see detailed import progress.
 
+## Development
+
+This project is built with TypeScript for enhanced type safety and developer experience.
+
+### Development Commands
+```bash
+# Build TypeScript to JavaScript
+npm run build
+
+# Build and watch for changes
+npm run build:watch
+
+# Run CLI commands (after building)
+npm run dev pull                    # Build and run pull command
+npm run pull                        # Build and run pull command
+npm run push                        # Build and run push command
+npm run status                      # Build and run status command
+
+# Type checking without emitting files
+npm run typecheck
+
+# Run tests
+npm test
+
+# Run tests with coverage reporting
+npm run test:coverage
+
+# Run specific test suites
+npm run test:unit          # Core module tests (api, sync, auth, hash, fsutil, akb)
+npm run test:integration   # End-to-end integration tests
+```
+
+### Test Coverage
+NEWO CLI includes comprehensive test suites:
+- **Authentication Tests** (`test/auth.test.js`): Token management, API key validation, multi-customer support
+- **Hashing Tests** (`test/hash.test.js`): SHA256 operations, hash storage, change detection
+- **File System Tests** (`test/fsutil.test.js`): Path utilities, directory management, atomic operations
+- **AKB Import Tests** (`test/akb.test.js`): Article parsing, Unicode handling, import workflows
+- **API Tests** (`test/api.test.js`): HTTP client functionality, NEWO API integration
+- **Sync Tests** (`test/sync.test.js`): Pull/push operations, project synchronization
+- **Integration Tests** (`test/integration.test.js`): End-to-end CLI functionality
+
+Test infrastructure includes:
+- **MockHttpClient**: HTTP request/response simulation
+- **MockFileSystem**: File system operation mocking
+- **TestEnvironment**: Isolated test environments with automatic cleanup
+- **Coverage Reporting**: HTML and text coverage reports via c8
+
+### Project Structure
+- `src/` - TypeScript source files
+- `dist/` - Compiled JavaScript output (generated by `npm run build`)
+- `test/` - Test files
+- `projects/` - Downloaded NEWO projects (generated by pull command)
+- `.newo/` - CLI state directory (tokens, hashes, mappings)
+
+### TypeScript Features
+- Full type safety with strict TypeScript configuration
+- Modern ES2022 target with ESNext modules
+- Comprehensive type definitions for all NEWO API responses
+- Enhanced error handling and validation
+- IntelliSense support in compatible IDEs
+
 ## API Endpoints
 - `GET /api/v1/designer/projects` - List all accessible projects
 - `GET /api/v1/designer/projects/by-id/{projectId}` - Get specific project metadata

package/dist/akb.d.ts

@@ -0,0 +1,10 @@
+import type { ParsedArticle, AkbImportArticle } from './types.js';
+/**
+ * Parse AKB file and extract articles
+ */
+export declare function parseAkbFile(filePath: string): Promise<ParsedArticle[]>;
+/**
+ * Convert parsed articles to API format for bulk import
+ */
+export declare function prepareArticlesForImport(articles: ParsedArticle[], personaId: string): AkbImportArticle[];
+//# sourceMappingURL=akb.d.ts.map

package/dist/akb.js

@@ -0,0 +1,88 @@
+import fs from 'fs-extra';
+/**
+ * Parse AKB file and extract articles
+ */
+export async function parseAkbFile(filePath) {
+    const content = await fs.readFile(filePath, 'utf8');
+    const articles = [];
+    // Split by article separators (---)
+    const sections = content.split(/^---\s*$/gm).filter(section => section.trim());
+    for (const section of sections) {
+        const lines = section.split('\n').filter(line => line.trim());
+        if (lines.length === 0)
+            continue;
+        const article = parseArticleSection(lines);
+        if (article) {
+            articles.push(article);
+        }
+    }
+    return articles;
+}
+/**
+ * Parse individual article section
+ */
+function parseArticleSection(lines) {
+    const state = {
+        topicName: '',
+        category: '',
+        summary: '',
+        keywords: '',
+        topicSummary: ''
+    };
+    // Find topic name (# r001)
+    const topicLine = lines.find(line => line.match(/^#\s+r\d+/));
+    if (!topicLine) {
+        console.warn('No topic line found in section');
+        return null;
+    }
+    state.topicName = topicLine.replace(/^#\s+/, '').trim();
+    // Extract category/subcategory/description (first ## line)
+    const categoryLine = lines.find(line => line.startsWith('## ') && line.includes(' / '));
+    if (categoryLine) {
+        state.category = categoryLine.replace(/^##\s+/, '').trim();
+    }
+    // Extract summary (second ## line)
+    const summaryLineIndex = lines.findIndex(line => line.startsWith('## ') && line.includes(' / '));
+    if (summaryLineIndex >= 0 && summaryLineIndex + 1 < lines.length) {
+        const nextLine = lines[summaryLineIndex + 1];
+        if (nextLine && nextLine.startsWith('## ') && !nextLine.includes(' / ')) {
+            state.summary = nextLine.replace(/^##\s+/, '').trim();
+        }
+    }
+    // Extract keywords (third ## line)
+    const keywordsLineIndex = lines.findIndex((line, index) => index > summaryLineIndex + 1 && line.startsWith('## ') && !line.includes(' / '));
+    if (keywordsLineIndex >= 0) {
+        const keywordsLine = lines[keywordsLineIndex];
+        if (keywordsLine) {
+            state.keywords = keywordsLine.replace(/^##\s+/, '').trim();
+        }
+    }
+    // Extract category content
+    const categoryStartIndex = lines.findIndex(line => line.includes('<Category type='));
+    const categoryEndIndex = lines.findIndex(line => line.includes('</Category>'));
+    if (categoryStartIndex >= 0 && categoryEndIndex >= 0) {
+        const categoryLines = lines.slice(categoryStartIndex, categoryEndIndex + 1);
+        state.topicSummary = categoryLines.join('\n');
+    }
+    // Create topic_facts array
+    const topicFacts = [state.category, state.summary, state.keywords].filter(fact => fact.trim() !== '');
+    return {
+        topic_name: state.category, // Use the descriptive title as topic_name
+        persona_id: null, // Will be set when importing
+        topic_summary: state.topicSummary,
+        topic_facts: topicFacts,
+        confidence: 100,
+        source: state.topicName, // Use the ID (r001) as source
+        labels: ['rag_context']
+    };
+}
+/**
+ * Convert parsed articles to API format for bulk import
+ */
+export function prepareArticlesForImport(articles, personaId) {
+    return articles.map(article => ({
+        ...article,
+        persona_id: personaId
+    }));
+}
+//# sourceMappingURL=akb.js.map

package/dist/api.d.ts

@@ -0,0 +1,14 @@
+import { type AxiosInstance } from 'axios';
+import type { ProjectMeta, Agent, Skill, FlowEvent, FlowState, AkbImportArticle, CustomerProfile } from './types.js';
+export declare function makeClient(verbose?: boolean, token?: string): Promise<AxiosInstance>;
+export declare function listProjects(client: AxiosInstance): Promise<ProjectMeta[]>;
+export declare function listAgents(client: AxiosInstance, projectId: string): Promise<Agent[]>;
+export declare function getProjectMeta(client: AxiosInstance, projectId: string): Promise<ProjectMeta>;
+export declare function listFlowSkills(client: AxiosInstance, flowId: string): Promise<Skill[]>;
+export declare function getSkill(client: AxiosInstance, skillId: string): Promise<Skill>;
+export declare function updateSkill(client: AxiosInstance, skillObject: Skill): Promise<void>;
+export declare function listFlowEvents(client: AxiosInstance, flowId: string): Promise<FlowEvent[]>;
+export declare function listFlowStates(client: AxiosInstance, flowId: string): Promise<FlowState[]>;
+export declare function importAkbArticle(client: AxiosInstance, articleData: AkbImportArticle): Promise<unknown>;
+export declare function getCustomerProfile(client: AxiosInstance): Promise<CustomerProfile>;
+//# sourceMappingURL=api.d.ts.map

package/dist/api.js

@@ -0,0 +1,103 @@
+import axios, {} from 'axios';
+import { getValidAccessToken, forceReauth } from './auth.js';
+import { ENV } from './env.js';
+// Per-request retry tracking to avoid shared state issues
+const RETRY_SYMBOL = Symbol('retried');
+export async function makeClient(verbose = false, token) {
+    let accessToken = token || await getValidAccessToken();
+    if (verbose)
+        console.log('✓ Access token obtained');
+    const client = axios.create({
+        baseURL: ENV.NEWO_BASE_URL,
+        headers: { accept: 'application/json' }
+    });
+    client.interceptors.request.use(async (config) => {
+        config.headers = config.headers || {};
+        config.headers.Authorization = `Bearer ${accessToken}`;
+        if (verbose) {
+            console.log(`→ ${config.method?.toUpperCase()} ${config.url}`);
+            if (config.data)
+                console.log('  Data:', JSON.stringify(config.data, null, 2));
+            if (config.params)
+                console.log('  Params:', config.params);
+        }
+        return config;
+    });
+    client.interceptors.response.use((response) => {
+        if (verbose) {
+            console.log(`← ${response.status} ${response.config.method?.toUpperCase()} ${response.config.url}`);
+            if (response.data && Object.keys(response.data).length < 20) {
+                console.log('  Response:', JSON.stringify(response.data, null, 2));
+            }
+            else if (response.data) {
+                const itemCount = Array.isArray(response.data) ? response.data.length : Object.keys(response.data).length;
+                console.log(`  Response: [${typeof response.data}] ${Array.isArray(response.data) ? itemCount + ' items' : 'large object'}`);
+            }
+        }
+        return response;
+    }, async (error) => {
+        const status = error?.response?.status;
+        if (verbose) {
+            console.log(`← ${status} ${error.config?.method?.toUpperCase()} ${error.config?.url} - ${error.message}`);
+            if (error.response?.data)
+                console.log('  Error data:', error.response.data);
+        }
+        // Use per-request retry tracking to avoid shared state issues
+        const config = error.config;
+        if (status === 401 && !config?.[RETRY_SYMBOL]) {
+            if (config) {
+                config[RETRY_SYMBOL] = true;
+                if (verbose)
+                    console.log('🔄 Retrying with fresh token...');
+                accessToken = await forceReauth();
+                config.headers = config.headers || {};
+                config.headers.Authorization = `Bearer ${accessToken}`;
+                return client.request(config);
+            }
+        }
+        throw error;
+    });
+    return client;
+}
+export async function listProjects(client) {
+    const response = await client.get('/api/v1/designer/projects');
+    return response.data;
+}
+export async function listAgents(client, projectId) {
+    const response = await client.get('/api/v1/bff/agents/list', {
+        params: { project_id: projectId }
+    });
+    return response.data;
+}
+export async function getProjectMeta(client, projectId) {
+    const response = await client.get(`/api/v1/designer/projects/by-id/${projectId}`);
+    return response.data;
+}
+export async function listFlowSkills(client, flowId) {
+    const response = await client.get(`/api/v1/designer/flows/${flowId}/skills`);
+    return response.data;
+}
+export async function getSkill(client, skillId) {
+    const response = await client.get(`/api/v1/designer/skills/${skillId}`);
+    return response.data;
+}
+export async function updateSkill(client, skillObject) {
+    await client.put(`/api/v1/designer/flows/skills/${skillObject.id}`, skillObject);
+}
+export async function listFlowEvents(client, flowId) {
+    const response = await client.get(`/api/v1/designer/flows/${flowId}/events`);
+    return response.data;
+}
+export async function listFlowStates(client, flowId) {
+    const response = await client.get(`/api/v1/designer/flows/${flowId}/states`);
+    return response.data;
+}
+export async function importAkbArticle(client, articleData) {
+    const response = await client.post('/api/v1/akb/append-manual', articleData);
+    return response.data;
+}
+export async function getCustomerProfile(client) {
+    const response = await client.get('/api/v1/customer/profile');
+    return response.data;
+}
+//# sourceMappingURL=api.js.map

package/dist/auth.d.ts

@@ -0,0 +1,6 @@
+import type { StoredTokens, CustomerConfig } from './types.js';
+export declare function exchangeApiKeyForToken(customer?: CustomerConfig): Promise<StoredTokens>;
+export declare function refreshWithEndpoint(refreshToken: string, customer?: CustomerConfig): Promise<StoredTokens>;
+export declare function getValidAccessToken(customer?: CustomerConfig): Promise<string>;
+export declare function forceReauth(customer?: CustomerConfig): Promise<string>;
+//# sourceMappingURL=auth.d.ts.map