doc2vec 2.8.0 → 2.9.1

package/README.md

@@ -229,6 +229,8 @@ Configuration is managed through two files:
         *   `encoding`: (Optional) Text file encoding (defaults to `'utf8'`). Does not apply to binary files (PDF, DOC, DOCX).
         *   `url_rewrite_prefix`: (Optional) URL prefix to rewrite `s3://` URLs (e.g., `'https://docs.example.com'`).
 
+        **S3 user metadata resolution:** The `product_name` and `version` fields support a `metadata(...)` syntax to dynamically resolve values from S3 object user metadata. For example, `product_name: 'metadata(x-amz-meta-product-name)'` will set `product_name` to the value of the `x-amz-meta-product-name` user metadata on each S3 object. If the metadata key doesn't exist on an object, an empty string is used. Literal values (without the `metadata(...)` wrapper) work as before.
+
         Authentication uses the AWS SDK default credential chain: environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`), `~/.aws/credentials`, IAM roles, etc.
 
         Incremental sync tracks object `LastModified` timestamps so only new or updated objects are processed on subsequent runs. Deleted objects are automatically cleaned up.
@@ -365,7 +367,7 @@ Configuration is managed through two files:
       
       # S3 bucket source example
       - type: 's3'
-        product_name: 'my-docs'
+        product_name: 'metadata(x-amz-meta-product-name)'
         version: 'latest'
         bucket: 'my-documentation-bucket'
         prefix: 'docs/v2/'

package/dist/content-processor.js

@@ -1707,7 +1707,10 @@ class ContentProcessor {
         if (lang) {
             try {
                 const codeChunker = await this.getCodeChunker(lang, sourceConfig.chunk_size);
-                chunks = await codeChunker.chunk(code);
+                chunks = await Promise.race([
+                    codeChunker.chunk(code),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error(`CodeChunker timed out after 30s for ${normalizedPath || url}`)), 30000))
+                ]);
             }
             catch (error) {
                 logger.warn(`CodeChunker failed for ${normalizedPath || url}, falling back to token chunking:`, error);

package/dist/doc2vec.js

@@ -783,7 +783,14 @@ class Doc2Vec {
                 else {
                     fileUrl = `s3://${config.bucket}/${obj.key}`;
                 }
-                const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
+                // Resolve metadata(...) references in product_name and version
+                const s3Meta = getResponse.Metadata || {};
+                const resolvedConfig = {
+                    ...config,
+                    product_name: this.resolveS3MetadataValue(config.product_name, s3Meta),
+                    version: this.resolveS3MetadataValue(config.version, s3Meta),
+                };
+                const chunks = await this.contentProcessor.chunkMarkdown(content, resolvedConfig, fileUrl);
                 logger.info(`Created ${chunks.length} chunks for ${obj.key}`);
                 await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
                 processedCount++;
@@ -828,6 +835,21 @@ class Doc2Vec {
         await database_1.DatabaseManager.setMetadataValue(dbConnection, lastSyncKey, `${syncStartTimestamp}`, logger, this.embeddingDimension);
         logger.info(`Finished processing S3 bucket: ${config.bucket}`);
     }
+    /**
+     * Resolves a config value that may use the metadata(...) syntax.
+     * e.g. "metadata(x-amz-meta-product-name)" looks up "product-name" in the S3 object's user metadata.
+     * Returns the original value if no metadata(...) pattern is found.
+     * Returns empty string if the referenced metadata key doesn't exist on the object.
+     */
+    resolveS3MetadataValue(configValue, s3Metadata) {
+        const match = configValue.match(/^metadata\((.+)\)$/);
+        if (!match)
+            return configValue;
+        const metaKey = match[1];
+        // AWS SDK returns user metadata keys without the x-amz-meta- prefix
+        const lookupKey = metaKey.replace(/^x-amz-meta-/, '');
+        return s3Metadata[lookupKey] ?? '';
+    }
     async processCodeSource(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for code source (${config.source})`);
@@ -1508,7 +1530,7 @@ class Doc2Vec {
             const response = await this.openai.embeddings.create({
                 model: this.embeddingModel,
                 input: safeTexts,
-            });
+            }, { timeout: 60000 });
             logger.debug(`Successfully created ${response.data.length} embeddings`);
             return response.data.map(d => d.embedding);
         }
@@ -1533,5 +1555,7 @@ if (require.main === module) {
         process.exit(1);
     }
     const doc2Vec = new Doc2Vec(configPath);
-    doc2Vec.run().catch(console.error);
+    doc2Vec.run()
+        .then(() => process.exit(0))
+        .catch((err) => { console.error(err); process.exit(1); });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.8.0",
+    "version": "2.9.1",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",