doc2vec 1.0.0 → 1.0.2

package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/lib/acm-cdk-stack.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AcmCdkStack = void 0;
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+class AcmCdkStack extends aws_cdk_lib_1.Stack {
+    constructor(scope, id, props) {
+        super(scope, id, props);
+        if (!process.env.DOMAIN_NAME) {
+            throw new Error("DOMAIN_NAME environment variable is not defined");
+        }
+        const edgeSiteDomainName = "edge." + process.env.DOMAIN_NAME;
+        const zone = aws_cdk_lib_1.aws_route53.HostedZone.fromLookup(this, 'Zone', { domainName: process.env.DOMAIN_NAME });
+        // setup acm
+        this.acmCertificate = new aws_cdk_lib_1.aws_certificatemanager.Certificate(this, 'SiteCertificate', {
+            domainName: edgeSiteDomainName,
+            validation: aws_cdk_lib_1.aws_certificatemanager.CertificateValidation.fromDns(zone),
+        });
+        new aws_cdk_lib_1.CfnOutput(this, 'ACMCertificate', { value: this.acmCertificate.certificateArn });
+    }
+}
+exports.AcmCdkStack = AcmCdkStack;

package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/lib/s3-cloudfront-cdk-stack.js

@@ -0,0 +1,95 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3CloudfrontCdkStack = void 0;
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+class S3CloudfrontCdkStack extends aws_cdk_lib_1.Stack {
+    constructor(scope, id, props) {
+        super(scope, id, props);
+        const acmCertificate = props.acmCertificate;
+        const originBucketUriHeader = 'x-s3-bucket-host';
+        const bucketName = process.env.S3_BUCKET_NAME || "default-s3-bucket";
+        const bucketUri = process.env.S3_BUCKET_URI;
+        if (!process.env.DOMAIN_NAME) {
+            throw new Error("DOMAIN_NAME environment variable is not defined");
+        }
+        const edgeSiteDomainName = "edge." + process.env.DOMAIN_NAME;
+        const originDomainName = "apps." + process.env.DOMAIN_NAME;
+        if (!bucketUri) {
+            throw new Error("S3_BUCKET_URI environment variable is not defined");
+        }
+        const zone = aws_cdk_lib_1.aws_route53.HostedZone.fromLookup(this, 'Zone', { domainName: process.env.DOMAIN_NAME });
+        // create an S3 bucket
+        const bucket = new aws_cdk_lib_1.aws_s3.Bucket(this, 'PrivateBucket', {
+            bucketName: bucketName,
+            removalPolicy: aws_cdk_lib_1.RemovalPolicy.DESTROY,
+            autoDeleteObjects: true,
+            publicReadAccess: false,
+            blockPublicAccess: new aws_cdk_lib_1.aws_s3.BlockPublicAccess({
+                blockPublicAcls: false,
+                blockPublicPolicy: false,
+                ignorePublicAcls: false,
+                restrictPublicBuckets: false
+            }),
+            versioned: true,
+        });
+        // attach the policy to only credentials based on SigV4
+        let authHeaderPolicy = new aws_cdk_lib_1.aws_iam.PolicyStatement({
+            actions: ['s3:Get*'],
+            resources: [bucket.arnForObjects('*')],
+            principals: [new aws_cdk_lib_1.aws_iam.AnyPrincipal()]
+        });
+        authHeaderPolicy.addCondition('StringEquals', {
+            's3:signatureversion': 'AWS4-HMAC-SHA256'
+        });
+        authHeaderPolicy.addCondition('StringEquals', {
+            's3:authType': 'REST-HEADER'
+        });
+        bucket.addToResourcePolicy(authHeaderPolicy);
+        // deploy all the artifacts
+        new aws_cdk_lib_1.aws_s3_deployment.BucketDeployment(this, 'DeployToBucket', {
+            sources: [aws_cdk_lib_1.aws_s3_deployment.Source.asset('./s3-bucket-contents/original')],
+            destinationBucket: bucket,
+        });
+        // setup cloudfront
+        const cdnDistribution = new aws_cdk_lib_1.aws_cloudfront.Distribution(this, 'SiteCDN', {
+            domainNames: [edgeSiteDomainName],
+            certificate: acmCertificate,
+            minimumProtocolVersion: aws_cdk_lib_1.aws_cloudfront.SecurityPolicyProtocol.TLS_V1_2_2021,
+            enableIpv6: false,
+            defaultBehavior: {
+                origin: new aws_cdk_lib_1.aws_cloudfront_origins.HttpOrigin(originDomainName, {
+                    protocolPolicy: aws_cdk_lib_1.aws_cloudfront.OriginProtocolPolicy.HTTPS_ONLY,
+                    originSslProtocols: [aws_cdk_lib_1.aws_cloudfront.OriginSslPolicy.TLS_V1_2],
+                    httpsPort: 443,
+                    readTimeout: aws_cdk_lib_1.Duration.seconds(45),
+                    connectionTimeout: aws_cdk_lib_1.Duration.seconds(10),
+                    customHeaders: {
+                        'x-s3-bucket-host': bucketUri
+                    }
+                }),
+                viewerProtocolPolicy: aws_cdk_lib_1.aws_cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS,
+                allowedMethods: aws_cdk_lib_1.aws_cloudfront.AllowedMethods.ALLOW_GET_HEAD_OPTIONS,
+                cachedMethods: aws_cdk_lib_1.aws_cloudfront.CachedMethods.CACHE_GET_HEAD_OPTIONS,
+                cachePolicy: aws_cdk_lib_1.aws_cloudfront.CachePolicy.CACHING_OPTIMIZED,
+                compress: true
+            }
+        });
+        // add A record to route53
+        new aws_cdk_lib_1.aws_route53.ARecord(this, 'SiteAliasARecord', {
+            recordName: edgeSiteDomainName,
+            target: aws_cdk_lib_1.aws_route53.RecordTarget.fromAlias(new aws_cdk_lib_1.aws_route53_targets.CloudFrontTarget(cdnDistribution)),
+            zone
+        });
+        // output information
+        new aws_cdk_lib_1.CfnOutput(this, 'SiteDistribution', {
+            value: cdnDistribution.distributionId,
+        });
+        new aws_cdk_lib_1.CfnOutput(this, 'BucketDomain', {
+            value: bucket.bucketRegionalDomainName,
+        });
+        new aws_cdk_lib_1.CfnOutput(this, 'SiteDomain', {
+            value: cdnDistribution.domainName,
+        });
+    }
+}
+exports.S3CloudfrontCdkStack = S3CloudfrontCdkStack;

package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/test/s3-cloudfront-cdk.test.js

@@ -0,0 +1,16 @@
+"use strict";
+// import * as cdk from 'aws-cdk-lib';
+// import { Template } from 'aws-cdk-lib/assertions';
+// import * as S3CloudfrontCdk from '../lib/s3-cloudfront-cdk-stack';
+// example test. To run these tests, uncomment this file along with the
+// example resource in lib/s3-cloudfront-cdk-stack.ts
+test('SQS Queue Created', () => {
+    //   const app = new cdk.App();
+    //     // WHEN
+    //   const stack = new S3CloudfrontCdk.S3CloudfrontCdkStack(app, 'MyTestStack');
+    //     // THEN
+    //   const template = Template.fromStack(stack);
+    //   template.hasResourceProperties('AWS::SQS::Queue', {
+    //     VisibilityTimeout: 300
+    //   });
+});

package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/bin/acm-cdk.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+require("source-map-support/register");
+const cdk = __importStar(require("aws-cdk-lib"));
+const acm_cdk_stack_1 = require("../lib/acm-cdk-stack");
+const route53_cdk_stack_1 = require("../lib/route53-cdk-stack");
+const app = new cdk.App();
+const route53Stack = new route53_cdk_stack_1.Route53CdkStack(app, 'Route53CdkStack', {
+    env: {
+        account: process.env.CDK_DEFAULT_ACCOUNT,
+        region: process.env.CLUSTER_REGION || process.env.CDK_DEFAULT_REGION
+    }
+});
+const acmStack = new acm_cdk_stack_1.ACMCdkStack(app, 'ACMCdkStack', {
+    env: {
+        account: process.env.CDK_DEFAULT_ACCOUNT,
+        region: process.env.CLUSTER_REGION || process.env.CDK_DEFAULT_REGION
+    },
+    hostedZone: route53Stack.hostedZone
+});
+acmStack.addDependency(route53Stack);

package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/lib/acm-cdk-stack.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ACMCdkStack = void 0;
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+class ACMCdkStack extends aws_cdk_lib_1.Stack {
+    constructor(scope, id, props) {
+        super(scope, id, props);
+        if (!process.env.DOMAIN_NAME) {
+            throw new Error("DOMAIN_NAME environment variable is not defined");
+        }
+        const hostedZone = props.hostedZone;
+        const edgeSiteDomainName = "apps." + process.env.DOMAIN_NAME;
+        this.acmCertificate = new aws_cdk_lib_1.aws_certificatemanager.Certificate(this, 'SiteCertificate', {
+            domainName: edgeSiteDomainName,
+            validation: aws_cdk_lib_1.aws_certificatemanager.CertificateValidation.fromDns(hostedZone),
+        });
+        new aws_cdk_lib_1.CfnOutput(this, 'ACMCertificate', { value: this.acmCertificate.certificateArn });
+    }
+}
+exports.ACMCdkStack = ACMCdkStack;

package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/lib/route53-cdk-stack.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Route53CdkStack = void 0;
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+class Route53CdkStack extends aws_cdk_lib_1.Stack {
+    constructor(scope, id, props) {
+        super(scope, id, props);
+        if (!process.env.PARENT_DOMAIN_NAME) {
+            throw new Error("PARENT_DOMAIN_NAME environment variable is not defined");
+        }
+        if (!process.env.DOMAIN_NAME) {
+            throw new Error("DOMAIN_NAME environment variable is not defined");
+        }
+        this.hostedZone = new aws_cdk_lib_1.aws_route53.PublicHostedZone(this, "HostedZone", {
+            zoneName: process.env.DOMAIN_NAME,
+            comment: "Managed by CDK"
+        });
+        const parent = aws_cdk_lib_1.aws_route53.HostedZone.fromLookup(this, "ParentHostedZone", {
+            domainName: process.env.PARENT_DOMAIN_NAME + "."
+        });
+        if (!this.hostedZone.hostedZoneNameServers) {
+            throw new Error("NS record is undefined.");
+        }
+        new aws_cdk_lib_1.aws_route53.ZoneDelegationRecord(this, "NS", {
+            zone: parent,
+            recordName: process.env.DOMAIN_NAME,
+            nameServers: this.hostedZone.hostedZoneNameServers,
+            comment: "Managed by CDK"
+        });
+        new aws_cdk_lib_1.CfnOutput(this, 'ZoneId', { value: this.hostedZone.hostedZoneId });
+    }
+}
+exports.Route53CdkStack = Route53CdkStack;

package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/test/s3-cloudfront-cdk.test.js

@@ -0,0 +1,16 @@
+"use strict";
+// import * as cdk from 'aws-cdk-lib';
+// import { Template } from 'aws-cdk-lib/assertions';
+// import * as S3CloudfrontCdk from '../lib/s3-cloudfront-cdk-stack';
+// example test. To run these tests, uncomment this file along with the
+// example resource in lib/s3-cloudfront-cdk-stack.ts
+test('SQS Queue Created', () => {
+    //   const app = new cdk.App();
+    //     // WHEN
+    //   const stack = new S3CloudfrontCdk.S3CloudfrontCdkStack(app, 'MyTestStack');
+    //     // THEN
+    //   const template = Template.fromStack(stack);
+    //   template.hasResourceProperties('AWS::SQS::Queue', {
+    //     VisibilityTimeout: 300
+    //   });
+});

package/doc2vec.ts

@@ -43,6 +43,7 @@ interface LocalDirectorySourceConfig extends BaseSourceConfig {
     exclude_extensions?: string[]; // File extensions to exclude
     recursive?: boolean;           // Whether to traverse subdirectories
     encoding?: BufferEncoding;     // File encoding (default: 'utf8')
+    url_rewrite_prefix?: string;   // Optional URL prefix to rewrite file:// URLs (e.g., 'https://mydomain.com')
 }
 
 // Configuration specific to website sources
@@ -725,8 +726,33 @@ class Doc2Vec {
                 
                 logger.info(`Processing content from ${filePath} (${content.length} chars)`);
                 try {
-                    // Use the file path as URL for tracking
-                    const fileUrl = `file://${filePath}`;
+                    // Generate URL based on configuration
+                    let fileUrl: string;
+                    
+                    if (config.url_rewrite_prefix) {
+                        // Replace local path with URL prefix
+                        const relativePath = path.relative(config.path, filePath).replace(/\\/g, '/');
+                        
+                        // If relativePath starts with '..', it means the file is outside the base directory
+                        if (relativePath.startsWith('..')) {
+                            // For files outside the configured path, use the default file:// scheme
+                            fileUrl = `file://${filePath}`;
+                            logger.debug(`File outside configured path, using default URL: ${fileUrl}`);
+                        } else {
+                            // For files inside the configured path, rewrite the URL
+                            // Handle trailing slashes in the URL prefix to avoid double slashes
+                            const prefix = config.url_rewrite_prefix.endsWith('/') 
+                                ? config.url_rewrite_prefix.slice(0, -1) 
+                                : config.url_rewrite_prefix;
+                                
+                            fileUrl = `${prefix}/${relativePath}`;
+                            logger.debug(`URL rewritten: ${filePath} -> ${fileUrl}`);
+                        }
+                    } else {
+                        // Use default file:// URL
+                        fileUrl = `file://${filePath}`;
+                    }
+                    
                     const chunks = await this.chunkMarkdown(content, config, fileUrl);
                     logger.info(`Created ${chunks.length} chunks`);
                     
@@ -806,15 +832,13 @@ class Doc2Vec {
             logger
         );
         
-        logger.info(`Found ${validChunkIds.size} valid chunks across processed files for ${config.path}`);
-        
         logger.section('CLEANUP');
         if (dbConnection.type === 'sqlite') {
             logger.info(`Running SQLite cleanup for local directory ${config.path}`);
-            this.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config.path, logger);
+            this.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
         } else if (dbConnection.type === 'qdrant') {
             logger.info(`Running Qdrant cleanup for local directory ${config.path} in collection ${dbConnection.collectionName}`);
-            await this.removeObsoleteFilesQdrant(dbConnection, processedFiles, config.path, logger);
+            await this.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
         }
         
         logger.info(`Finished processing local directory: ${config.path}`);
@@ -921,6 +945,160 @@ class Doc2Vec {
         }
     }
 
+    private removeObsoleteFilesSQLite(
+        db: Database, 
+        processedFiles: Set<string>, 
+        pathConfig: { path: string; url_rewrite_prefix?: string } | string, 
+        logger: Logger
+    ) {
+        const getChunksForPathStmt = db.prepare(`
+            SELECT chunk_id, url FROM vec_items
+            WHERE url LIKE ? || '%'
+        `);
+        const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
+        
+        // Determine if we're using URL rewriting or direct file paths
+        const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
+        
+        // Set up the URL prefix for searching
+        let urlPrefix: string;
+        if (isRewriteMode) {
+            // Handle URL rewriting case
+            urlPrefix = (pathConfig as { path: string; url_rewrite_prefix?: string }).url_rewrite_prefix || '';
+            urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
+        } else {
+            // Handle direct file path case
+            const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
+            const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
+            urlPrefix = `file://${cleanedDirPrefix}`;
+        }
+        
+        logger.debug(`Searching for chunks with URL prefix: ${urlPrefix}`);
+        const existingChunks = getChunksForPathStmt.all(urlPrefix) as { chunk_id: string; url: string }[];
+        let deletedCount = 0;
+        
+        const transaction = db.transaction(() => {
+            for (const { chunk_id, url } of existingChunks) {
+                // Skip if it's not from our URL prefix (safety check)
+                if (!url.startsWith(urlPrefix)) continue;
+                
+                let filePath: string;
+                let shouldDelete = false;
+                
+                if (isRewriteMode) {
+                    // URL rewrite mode: extract relative path and construct full file path
+                    const config = pathConfig as { path: string; url_rewrite_prefix?: string };
+                    const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
+                    filePath = path.join(config.path, relativePath);
+                    shouldDelete = !processedFiles.has(filePath);
+                } else {
+                    // Direct file path mode: remove file:// prefix to match with processedFiles
+                    filePath = url.substring(7); // Remove 'file://' prefix
+                    shouldDelete = !processedFiles.has(filePath);
+                }
+                
+                if (shouldDelete) {
+                    logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed: ${filePath})`);
+                    deleteChunkStmt.run(chunk_id);
+                    deletedCount++;
+                }
+            }
+        });
+        transaction();
+        
+        logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL prefix ${urlPrefix}`);
+    }
+
+    private async removeObsoleteFilesQdrant(
+        db: QdrantDB, 
+        processedFiles: Set<string>, 
+        pathConfig: { path: string; url_rewrite_prefix?: string } | string, 
+        logger: Logger
+    ) {
+        const { client, collectionName } = db;
+        try {
+            // Determine if we're using URL rewriting or direct file paths
+            const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
+            
+            // Set up the URL prefix for searching
+            let urlPrefix: string;
+            if (isRewriteMode) {
+                // Handle URL rewriting case
+                urlPrefix = (pathConfig as { path: string; url_rewrite_prefix?: string }).url_rewrite_prefix || '';
+                urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
+            } else {
+                // Handle direct file path case
+                const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
+                const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
+                urlPrefix = `file://${cleanedDirPrefix}`;
+            }
+            
+            logger.debug(`Checking for obsolete chunks with URL prefix: ${urlPrefix}`);
+            const response = await client.scroll(collectionName, {
+                limit: 10000,
+                with_payload: true,
+                with_vector: false,
+                filter: {
+                    must: [
+                        {
+                            key: "url",
+                            match: {
+                                text: urlPrefix
+                            }
+                        }
+                    ],
+                    must_not: [
+                        {
+                            key: "is_metadata",
+                            match: {
+                                value: true
+                            }
+                        }
+                    ]
+                }
+            });
+            
+            const obsoletePointIds = response.points
+                .filter((point: any) => {
+                    const url = point.payload?.url;
+                    // Double check it's not a metadata record
+                    if (point.payload?.is_metadata === true) {
+                        return false;
+                    }
+                    
+                    if (!url || !url.startsWith(urlPrefix)) {
+                        return false;
+                    }
+                    
+                    let filePath: string;
+                    
+                    if (isRewriteMode) {
+                        // URL rewrite mode: extract relative path and construct full file path
+                        const config = pathConfig as { path: string; url_rewrite_prefix?: string };
+                        const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
+                        filePath = path.join(config.path, relativePath);
+                    } else {
+                        // Direct file path mode: remove file:// prefix to match with processedFiles
+                        filePath = url.startsWith('file://') ? url.substring(7) : '';
+                    }
+                    
+                    return filePath && !processedFiles.has(filePath);
+                })
+                .map((point: any) => point.id);
+            
+            if (obsoletePointIds.length > 0) {
+                await client.delete(collectionName, {
+                    points: obsoletePointIds,
+                });
+                logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for URL prefix ${urlPrefix}`);
+            } else {
+                logger.info(`No obsolete chunks to delete from Qdrant for URL prefix ${urlPrefix}`);
+            }
+        } catch (error) {
+            logger.error(`Error removing obsolete chunks from Qdrant:`, error);
+        }
+    }
+
     private async initDatabase(config: SourceConfig, parentLogger: Logger): Promise<DatabaseConnection> {
         const logger = parentLogger.child('database');
         const dbConfig = config.database_config;
@@ -1116,93 +1294,6 @@ class Doc2Vec {
     
         logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL ${urlPrefix}`);
     }
-
-    private removeObsoleteFilesSQLite(db: Database, processedFiles: Set<string>, dirPrefix: string, logger: Logger) {
-        const getChunksForDirStmt = db.prepare(`
-            SELECT chunk_id, url FROM vec_items
-            WHERE url LIKE 'file://%' AND url LIKE ?
-        `);
-        const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
-        
-        const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
-        const filePrefix = `file://${cleanedDirPrefix}`;
-        
-        const existingChunks = getChunksForDirStmt.all(`${filePrefix}%`) as { chunk_id: string; url: string }[];
-        let deletedCount = 0;
-        
-        const transaction = db.transaction(() => {
-            for (const { chunk_id, url } of existingChunks) {
-                // Remove file:// prefix to match with processedFiles
-                const filePath = url.substring(7);
-                if (!processedFiles.has(filePath)) {
-                    logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed)`);
-                    deleteChunkStmt.run(chunk_id);
-                    deletedCount++;
-                }
-            }
-        });
-        transaction();
-        
-        logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for directory ${dirPrefix}`);
-    }
-    
-    private async removeObsoleteFilesQdrant(db: QdrantDB, processedFiles: Set<string>, dirPrefix: string, logger: Logger) {
-        const { client, collectionName } = db;
-        try {
-
-            const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
-            const filePrefix = `file://${cleanedDirPrefix}`;
-            
-            // Get all points that match the file prefix but are not metadata points
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: `${filePrefix}`
-                            }
-                        }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
-                        }
-                    ]
-                }
-            });
-            
-            const obsoletePointIds = response.points
-                .filter((point: any) => {
-                    const url = point.payload?.url;
-                    // Double check it's not a metadata record
-                    if (point.payload?.is_metadata === true) {
-                        return false;
-                    }
-                    // Remove file:// prefix to match with processedFiles
-                    const filePath = url && url.startsWith('file://') ? url.substring(7) : '';
-                    return filePath && !processedFiles.has(filePath);
-                })
-                .map((point: any) => point.id);
-            
-            if (obsoletePointIds.length > 0) {
-                await client.delete(collectionName, {
-                    points: obsoletePointIds,
-                });
-                logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for directory ${dirPrefix}`);
-            } else {
-                logger.info(`No obsolete chunks to delete from Qdrant for directory ${dirPrefix}`);
-            }
-        } catch (error) {
-            logger.error(`Error removing obsolete chunks from Qdrant:`, error);
-        }
-    }
     
     private isValidUuid(str: string): boolean {
         const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;