@j-o-r/hello-dave 0.0.2 → 0.0.4

package/lib/API/x.ai/collections.js

@@ -0,0 +1,354 @@
+/**
+ * [api documentation](https://docs.x.ai/docs/guides/using-collections/api)
+ */
+import { GLOBAL } from '../../fafs.js'
+import { request as doRequest } from '@j-o-r/apiserver';
+import FILES from './files.js';
+import fs from 'fs';
+import path from 'path';
+/**
+ * @typedef {Object} Collection
+ * @property {string} collection_id - Unique identifier for the collection.
+ * @property {string} collection_name - Name of the collection.
+ * @property {string} created_at - ISO datetime string when the collection was created.
+ * @property {string} updated_at
+ * @property {Object} index_configuration - Configuration for the indexing model.
+ * @property {string} index_configuration.model_name - Name of the embedding model.
+ * @property {Object} chunk_configuration - Configuration for chunking documents.
+ * @property {Object} chunk_configuration.tokens_configuration - Token-related chunk settings.
+ * @property {number} chunk_configuration.tokens_configuration.max_chunk_size_tokens - Maximum tokens per chunk.
+ * @property {number} chunk_configuration.tokens_configuration.chunk_overlap_tokens - Overlap tokens between chunks.
+ * @property {string} chunk_configuration.tokens_configuration.encoding_name - Token encoding name.
+ * @property {boolean} chunk_configuration.strip_whitespace - Whether to strip whitespace.
+ * @property {boolean} chunk_configuration.inject_name_into_chunks - Whether to inject name into chunks.
+ * @property {number} documents_count - Number of documents in the collection.
+ * @property {Array} field_definitions - Array of field definitions.
+ */
+ /**
+ * @typedef fileList
+ * @property {File[]} data
+ * @property {string|null} pagination_token
+ */
+/**
+ * @typedef {Object} File
+ * @property {number} bytes - File size in bytes.
+ * @property {number} created_at - Unix timestamp of creation.
+ * @property {number|null} expires_at - Unix timestamp of expiration, or null.
+ * @property {string} filename - Path to the file.
+ * @property {string} id - Unique file identifier.
+ * @property {string} object - Type of object, e.g., 'file'.
+ * @property {string} purpose - Purpose of the file.
+ *//**
+ * @typedef {Object} SearchResult
+ * @property {string} document_id
+ * @property {string} content
+ * @property {Object} metadata
+ * @property {number} score
+ */
+
+const API_BASE = 'https://api.x.ai/v1';
+const MANAGEMENT_BASE = 'https://management-api.x.ai/v1';
+
+
+const collectionStorage = new Map();
+
+/**
+ * Get the default headers for API
+ * @returns {object}
+ */
+const getAPIHeaders = () => {
+	if (!process.env['XAIKEY']) {
+		throw new Error('Missing XAIKEY! export XAIKEY=<XAIKEY>')
+	}
+	const KEY = process.env['XAIKEY'];
+	return {
+		'Authorization': `Bearer ${KEY}`
+	}
+}
+
+/**
+ * Get the default headers for Management API
+ * @returns {object}
+ */
+const getManagementHeaders = () => {
+	const KEY = process.env['XAI_MANAGEMENT_KEY'] || process.env['XAIKEY'];
+	if (!KEY) {
+		throw new Error('Missing XAI_MANAGEMENT_KEY or XAIKEY!')
+	}
+	return {
+		'Authorization': `Bearer ${KEY}`
+	}
+}
+
+/**
+ * Create a collection
+ * @param {string} name
+ * @returns {Promise<string>} collection_id
+ */
+async function create(name) {
+	const url = `${MANAGEMENT_BASE}/collections`;
+	const headers = { ...getManagementHeaders(), 'Content-Type': 'application/json' };
+	const body = { collection_name: name };
+	const res = await doRequest(url, 'POST', headers, body);
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response.collection_id;
+}
+
+/**
+ * List collections
+ * @returns {Promise<Array<Collection>>}
+ */
+async function list() {
+	const url = `${MANAGEMENT_BASE}/collections`;
+	const headers = getManagementHeaders();
+	const res = await doRequest(url, 'GET', headers, {});
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response.collections || [];
+}
+
+/**
+ * Get collection details
+ * @param {string} id
+ * @returns {Promise<Collection>}
+ */
+async function get(id) {
+	const url = `${MANAGEMENT_BASE}/collections/${id}`;
+	const headers = getManagementHeaders();
+	const res = await doRequest(url, 'GET', headers, {});
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response;
+}
+
+/**
+ * Update collection collection name
+ * @param {string} id
+ * @param {string} name
+ * @returns {Promise<void>}
+ */
+async function update(id, name) {
+	const url = `${MANAGEMENT_BASE}/collections/${id}`;
+	const headers = { ...getManagementHeaders(), 'Content-Type': 'application/json' };
+	const body = { collection_name: name };
+	const res = await doRequest(url, 'PUT', headers, body);
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+}
+
+/**
+ * Delete collection
+ * @param {string} id
+ * @returns {Promise<void>}
+ */
+async function del (id) {
+	const url = `${MANAGEMENT_BASE}/collections/${id}`;
+	const headers = getManagementHeaders();
+	const res = await doRequest(url, 'DELETE', headers, {});
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+}
+
+/**
+ * Add file to collection
+ * @param {string} collectionId
+ * @param {string} fileId
+ * @returns {Promise<void>}
+ */
+async function addFileToCollection(collectionId, fileId) {
+	const url = `${MANAGEMENT_BASE}/collections/${collectionId}/documents/${fileId}`;
+	const headers = getManagementHeaders();
+	const res = await doRequest(url, 'POST', headers, {});
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+}
+
+// /**
+//  * Upload and add file to collection with optional metadata
+//  * @param {string} collectionId
+//  * @param {string} filePath
+//  * @param {Object} [metadata]
+//  * @returns {Promise<string>} file_id
+//  */
+// async function uploadAndAddFileToCollection(collectionId, filePath, metadata = {}) {
+// 	const fileId = (await FILES.upload(filePath)).id;
+// 	const url = `${MANAGEMENT_BASE}/collections/${collectionId}/documents`;
+// 	const headers = getManagementHeaders();
+// 	const formData = new FormData();
+// 	formData.append('name', path.basename(filePath));
+// 	formData.append('data', fs.createReadStream(filePath));
+// 	formData.append('content_type', 'application/octet-stream'); // or detect
+// 	if (Object.keys(metadata).length > 0) {
+// 		formData.append('fields', JSON.stringify(metadata));
+// 	}
+// 	const response = await fetch(url, {
+// 		method: 'POST',
+// 		headers,
+// 		body: formData
+// 	});
+// 	if (!response.ok) {
+// 		throw new Error(`${response.status}: ${await response.text()}`);
+// 	}
+// 	const data = await response.json();
+// 	return fileId;
+// }
+
+// /**
+//  * Add files from folder to collection
+//  * @param {string} collectionId
+//  * @param {string} folderPath
+//  * @param {boolean} [recursive=false]
+//  * @param {Object} [metadata]
+//  * @returns {Promise<string[]>} file_ids
+//  */
+// async function addFilesFromFolder(collectionId, folderPath, recursive = false, metadata = {}) {
+// 	const files = [];
+// 	const readDir = (dir) => {
+// 		const items = fs.readdirSync(dir);
+// 		for (const item of items) {
+// 			const fullPath = path.join(dir, item);
+// 			const stat = fs.statSync(fullPath);
+// 			if (stat.isDirectory() && recursive) {
+// 				readDir(fullPath);
+// 			} else if (stat.isFile()) {
+// 				files.push(fullPath);
+// 			}
+// 		}
+// 	};
+// 	readDir(folderPath);
+// 	const fileIds = [];
+// 	for (const filePath of files) {
+// 		try {
+// 			const fileId = await uploadAndAddFileToCollection(collectionId, filePath, metadata);
+// 			fileIds.push(fileId);
+// 		} catch (error) {
+// 			console.error(`Failed to upload ${filePath}: ${error.message}`);
+// 		}
+// 	}
+// 	return fileIds;
+// }
+
+// /**
+//  * Resync folder to collection (delete all and re-add)
+//  * @param {string} collectionId
+//  * @param {string} folderPath
+//  * @param {boolean} [recursive=false]
+//  * @param {Object} [metadata]
+//  * @returns {Promise<string[]>} file_ids
+//  */
+// async function resyncFolder(collectionId, folderPath, recursive = false, metadata = {}) {
+// 	// First, delete all files in collection? But API doesn't have list files in collection directly.
+// 	// Perhaps list collections and delete them, but complicated.
+// 	// For simplicity, assume we don't delete, just add new.
+// 	// To resync, perhaps delete collection and recreate, but that would lose id.
+// 	// Since no direct way, maybe just add new files.
+// 	// Perhaps get collection details, but doesn't list files.
+// 	// For now, just add files, assuming no duplicates.
+// 	return addFilesFromFolder(collectionId, folderPath, recursive, metadata);
+// }
+
+/**
+ * Delete file from collection
+ * @param {string} collectionId
+ * @param {string} fileId
+ * @returns {Promise<void>}
+ */
+async function deleteFileFromCollection(collectionId, fileId) {
+	const url = `${MANAGEMENT_BASE}/collections/${collectionId}/documents/${fileId}`;
+	const headers = getManagementHeaders();
+	const res = await doRequest(url, 'DELETE', headers, {});
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+}
+
+/**
+ * The status of files in a collection
+ * @param {string} collectionId
+ * @returns {Promise<void>}
+ */
+async function status(collectionId) {
+	const url = `${MANAGEMENT_BASE}/collections/${collectionId}/documents`;
+	const headers = getManagementHeaders();
+	const res = await doRequest(url, 'GET', headers, {});
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response
+}
+
+/**
+ * The status of files in a collection
+ * @param {string} collectionId
+ * @param {string} fileId
+ * @returns {Promise<void>}
+ */
+async function details(collectionId,fileId) {
+	const url = `${MANAGEMENT_BASE}/collections/${collectionId}/documents/${fileId}`;
+	const headers = getManagementHeaders();
+	const res = await doRequest(url, 'GET', headers, {});
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response
+}
+/**
+ * Search in collections
+ * @param {string} query
+ * @param {string[]} collectionIds
+ * @param {Object} [options]
+ * @param {string} [options.retrieval_mode] - e.g., 'hybrid'
+ * @returns {Promise<SearchResult[]>}
+ */
+async function searchInCollections(query, collectionIds, options = {}) {
+	const url = `${API_BASE}/documents/search`;
+	const headers = { ...getAPIHeaders(), 'Content-Type': 'application/json' };
+	const body = {
+		query,
+		source: {
+			collection_ids: collectionIds
+		},
+		...options
+	};
+	const res = await doRequest(url, 'POST', headers, body);
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response.results || res.response;
+}
+
+/**
+* @returns {Promise<Map<string, Collection>>}
+*/
+const getStorage = async () => {
+	const lst = await list();
+	let i = 0;
+	const len = lst.length;
+	for(;i < len; i++) {
+		const col = lst[i];
+		// Decided to have the name as a key.
+		collectionStorage.set(col.collection_name, col);
+	}
+	return collectionStorage;
+}
+
+export default {
+	create,
+	list,
+	get,
+	status,
+	details,
+	update,
+	del,
+	addFileToCollection,
+	deleteFileFromCollection,
+	searchInCollections,
+	storage: getStorage
+};

package/lib/API/x.ai/files.js

@@ -0,0 +1,218 @@
+/**
+ * https://docs.x.ai/docs/guides/files
+ Limitations
+File size: Maximum 48 MB per file
+No batch requests: File attachments with document search are agentic requests and do not support batch mode (n > 1)
+Agentic models only: Requires models that support agentic tool calling (e.g., grok-4-fast, grok-4)
+Supported file formats:
+Plain text files (.txt)
+Markdown files (.md)
+Code files (.py, .js, .java, etc.)
+CSV files (.csv)
+JSON files (.json)
+PDF documents (.pdf)
+And many other text-based formats
+
+ */
+import { request as doRequest } from '@j-o-r/apiserver';
+import fs from 'fs';
+
+
+const fileStorage = new Map();
+
+/**
+* @typedef fileList
+* @property {File[]} data
+* @property {string|null} pagination_token
+*/
+/**
+ * @typedef {Object} File
+ * @property {number} bytes - File size in bytes.
+ * @property {number} created_at - Unix timestamp of creation.
+ * @property {number|null} expires_at - Unix timestamp of expiration, or null.
+ * @property {string} filename - Path to the file.
+ * @property {string} id - Unique file identifier.
+ * @property {string} object - Type of object, e.g., 'file'.
+ * @property {string} purpose - Purpose of the file.
+ */
+
+const API_BASE = 'https://api.x.ai/v1';
+/**
+* Is buffer filled with text?
+* @param {Buffer} buffer
+* @returns {boolean}
+*/
+function isTextBuffer(buffer) {
+	const text = buffer.toString('utf8');
+	const nonPrintable = text.replace(/[\x20-\x7E\t\r\n]/g, '').length;
+	return nonPrintable / buffer.length < 0.3; // 30% threshold
+}
+/**
+ * Get the default headers for API
+ * @returns {object}
+ */
+const getAPIHeaders = () => {
+	if (!process.env['XAIKEY']) {
+		throw new Error('Missing XAIKEY! export XAIKEY=<XAIKEY>')
+	}
+	const KEY = process.env['XAIKEY'];
+	return {
+		'Authorization': `Bearer ${KEY}`
+	}
+}
+
+/**
+ * Get the default headers for Management API
+ * @returns {object}
+ */
+const getManagementHeaders = () => {
+	const KEY = process.env['XAI_MANAGEMENT_KEY'] || process.env['XAIKEY'];
+	if (!KEY) {
+		throw new Error('Missing XAI_MANAGEMENT_KEY or XAIKEY!')
+	}
+	return {
+		'Authorization': `Bearer ${KEY}`
+	}
+}
+
+/**
+ * List Files
+ * @param {number} limit
+ * @param {'asc'|'desc'} order
+ * @param {'created_at'|'filename'| 'size'} sort_by
+ * @param {string} page (pagination_token)
+ * @retuns {Promise<fileList>}
+ */
+async function list(limit = 100, order = 'asc', sort_by = 'size', page = '') {
+	const url = `${API_BASE}/files?limit=${limit}&order=${order}&sort_by=${sort_by}&pagination_token=${page}`;
+	const headers = getAPIHeaders();
+	const res = await doRequest(url,
+		'GET',
+		headers
+	);
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response;
+}
+/**
+ * Upload a file
+ * When content is filled this will be used as file content
+ * @param {string} filePath
+ * @param {string} [content] - optional content for a non existing file
+ * @returns {Promise<File>} file_id
+ */
+async function upload(filePath, content) {
+	const url = `${API_BASE}/files`;
+	const headers = getAPIHeaders();
+	const formData = new FormData();
+	let buffer;
+	if (!content) {
+	  buffer = fs.readFileSync(filePath);
+	} else {
+		buffer = Buffer.from(content, 'utf8');
+	}
+	if (buffer.length > 50331648) {
+		throw new Error('File too large: exceeds 48 MB limit');
+	}
+	if (!isTextBuffer(buffer)) {
+		throw new Error('File is not plain text based');
+	}
+	formData.append('file', new Blob([buffer]), filePath);
+	const result = await doRequest(url,
+		'POST',
+		headers,
+		formData
+	);
+	if (result.status < 200 || result.status >= 300) {
+		throw new Error(`${result.status}: ${result.response}`);
+	}
+	const data = result.response;
+	return data;
+}
+
+/**
+ * Get file content
+ * @param {string} fileId
+ * @returns {Promise<*>}
+ */
+async function get(fileId) {
+	const url = `${API_BASE}/files/${fileId}/content`;
+	const headers = getAPIHeaders();
+	const res = await doRequest(url,
+		'GET',
+		headers
+	);
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	if (res.responseType === 'blob') {
+		return res.response.text();
+	}
+	return res.response;
+}
+
+/**
+ * Get file meta data
+ * @param {string} fileId
+ * @returns {Promise<File>}
+ */
+async function getMeta(fileId) {
+	const url = `${API_BASE}/files/${fileId}`;
+	const headers = getAPIHeaders();
+	const res = await doRequest(url,
+		'GET',
+		headers
+	);
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+	return res.response;
+}
+
+/**
+ * Remove a file
+ * @param {string} fileId
+ * @returns {Promise<void>}
+ */
+async function rm(fileId) {
+	const url = `${API_BASE}/files/${fileId}`;
+	const headers = getAPIHeaders();
+	const res = await doRequest(url,
+		'DELETE',
+		headers
+	);
+	if (res.status !== 200) {
+		throw new Error(`${res.status}: ${JSON.stringify(res.response)}`);
+	}
+}
+
+
+/**
+* Get all on-line files
+* @returns {Promise<Map<string, File>>}
+*
+*/
+const getStorage = async (page = '') => {
+	if (!page) page = '';
+	const lst = await list(100, 'asc', 'size', page);
+	let i = 0;
+	const len = lst.data.length;
+	for (; i < len; i++) {
+		// console.log(list.data[i]);
+		fileStorage.set(lst.data[i].filename, lst.data[i]);
+	}
+	if (lst.pagination_token) {
+		return await getStorage(lst.pagination_token);
+	}
+	return fileStorage;
+}
+
+export default {
+	list,
+	upload,
+	get,
+	getMeta,
+	rm,
+	storage: getStorage
+};