doc2vec 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +8 -17
- package/Dockerfile.old +19 -0
- package/README.md +21 -0
- package/config.yaml.new +239 -165
- package/config.yaml.ok +164 -0
- package/config.yaml.solo +251 -0
- package/dist/doc2vec.js +160 -84
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/lib/acm-cdk-stack.js +20 -0
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/lib/route53-cdk-stack.js +33 -0
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/lib/waf-cdk-stack.js +66 -0
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/test/s3-cloudfront-cdk.test.js +16 -0
- package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/lib/acm-cdk-stack.js +21 -0
- package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/lib/s3-cloudfront-cdk-stack.js +95 -0
- package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/test/s3-cloudfront-cdk.test.js +16 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/bin/acm-cdk.js +55 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/lib/acm-cdk-stack.js +20 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/lib/route53-cdk-stack.js +33 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/test/s3-cloudfront-cdk.test.js +16 -0
- package/doc2vec.ts +184 -93
- package/doc2vec.ts.ok +1730 -0
- package/package.json +4 -2
- package/tmp/README.md.old +1 -0
- package/tmp.db +0 -0
package/config.yaml.solo
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
# Doc2Vec Configuration
|
|
2
|
+
sources:
|
|
3
|
+
# Local directory Sources
|
|
4
|
+
- type: local_directory
|
|
5
|
+
product_name: 'solo-reference-architectures'
|
|
6
|
+
version: 'latest'
|
|
7
|
+
path: './solo-reference-architectures'
|
|
8
|
+
include_extensions: ['.md']
|
|
9
|
+
exclude_extensions: []
|
|
10
|
+
recursive: true
|
|
11
|
+
url_rewrite_prefix: https://github.com/solo-io/solo-reference-architectures/blob/main
|
|
12
|
+
max_size: 1048576
|
|
13
|
+
database_config:
|
|
14
|
+
type: 'qdrant'
|
|
15
|
+
params:
|
|
16
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
17
|
+
qdrant_port: 443
|
|
18
|
+
collection_name: 'github-solo-reference-architectures'
|
|
19
|
+
|
|
20
|
+
# GitHub Sources
|
|
21
|
+
- type: github
|
|
22
|
+
product_name: 'istio'
|
|
23
|
+
version: 'latest'
|
|
24
|
+
repo: 'istio/istio'
|
|
25
|
+
start_date: '2024-01-01'
|
|
26
|
+
max_size: 1048576
|
|
27
|
+
database_config:
|
|
28
|
+
type: 'qdrant'
|
|
29
|
+
params:
|
|
30
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
31
|
+
qdrant_port: 443
|
|
32
|
+
collection_name: 'github-istio'
|
|
33
|
+
|
|
34
|
+
- type: github
|
|
35
|
+
product_name: 'solo-projects'
|
|
36
|
+
version: 'latest'
|
|
37
|
+
repo: 'solo-io/solo-projects'
|
|
38
|
+
start_date: '2024-01-01'
|
|
39
|
+
max_size: 1048576
|
|
40
|
+
database_config:
|
|
41
|
+
type: 'qdrant'
|
|
42
|
+
params:
|
|
43
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
44
|
+
qdrant_port: 443
|
|
45
|
+
collection_name: 'github-solo-projects'
|
|
46
|
+
|
|
47
|
+
- type: github
|
|
48
|
+
product_name: 'gloo-mesh-enterprise'
|
|
49
|
+
version: 'latest'
|
|
50
|
+
repo: 'solo-io/gloo-mesh-enterprise'
|
|
51
|
+
start_date: '2024-01-01'
|
|
52
|
+
max_size: 1048576
|
|
53
|
+
database_config:
|
|
54
|
+
type: 'qdrant'
|
|
55
|
+
params:
|
|
56
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
57
|
+
qdrant_port: 443
|
|
58
|
+
collection_name: 'github-gloo-mesh-enterprise'
|
|
59
|
+
|
|
60
|
+
# Website Sources
|
|
61
|
+
- type: website
|
|
62
|
+
product_name: 'ambient'
|
|
63
|
+
version: 'latest'
|
|
64
|
+
url: 'https://ambientmesh.io/docs/'
|
|
65
|
+
max_size: 1048576
|
|
66
|
+
database_config:
|
|
67
|
+
type: 'qdrant'
|
|
68
|
+
params:
|
|
69
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
70
|
+
qdrant_port: 443
|
|
71
|
+
collection_name: 'ambient'
|
|
72
|
+
|
|
73
|
+
- type: website
|
|
74
|
+
product_name: 'argo'
|
|
75
|
+
version: 'latest'
|
|
76
|
+
url: 'https://argo-cd.readthedocs.io/en/stable/'
|
|
77
|
+
max_size: 1048576
|
|
78
|
+
database_config:
|
|
79
|
+
type: 'qdrant'
|
|
80
|
+
params:
|
|
81
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
82
|
+
qdrant_port: 443
|
|
83
|
+
collection_name: 'argo'
|
|
84
|
+
|
|
85
|
+
- type: website
|
|
86
|
+
product_name: 'argo-rollouts'
|
|
87
|
+
version: 'latest'
|
|
88
|
+
url: 'https://argoproj.github.io/argo-rollouts/installation/'
|
|
89
|
+
max_size: 1048576
|
|
90
|
+
database_config:
|
|
91
|
+
type: 'qdrant'
|
|
92
|
+
params:
|
|
93
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
94
|
+
qdrant_port: 443
|
|
95
|
+
collection_name: 'argo-rollouts'
|
|
96
|
+
|
|
97
|
+
- type: website
|
|
98
|
+
product_name: 'argo-rollouts'
|
|
99
|
+
version: 'latest'
|
|
100
|
+
url: 'https://rollouts-plugin-trafficrouter-gatewayapi.readthedocs.io/en/latest/'
|
|
101
|
+
max_size: 1048576
|
|
102
|
+
database_config:
|
|
103
|
+
type: 'qdrant'
|
|
104
|
+
params:
|
|
105
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
106
|
+
qdrant_port: 443
|
|
107
|
+
collection_name: 'argo-rollouts'
|
|
108
|
+
|
|
109
|
+
- type: website
|
|
110
|
+
product_name: 'cilium'
|
|
111
|
+
version: 'latest'
|
|
112
|
+
url: 'https://docs.cilium.io/en/stable/'
|
|
113
|
+
max_size: 1048576
|
|
114
|
+
database_config:
|
|
115
|
+
type: 'qdrant'
|
|
116
|
+
params:
|
|
117
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
118
|
+
qdrant_port: 443
|
|
119
|
+
collection_name: 'cilium'
|
|
120
|
+
|
|
121
|
+
- type: website
|
|
122
|
+
product_name: 'gateway-api'
|
|
123
|
+
version: 'latest'
|
|
124
|
+
url: 'https://gateway-api.sigs.k8s.io/'
|
|
125
|
+
max_size: 1048576
|
|
126
|
+
database_config:
|
|
127
|
+
type: 'qdrant'
|
|
128
|
+
params:
|
|
129
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
130
|
+
qdrant_port: 443
|
|
131
|
+
collection_name: 'gateway-api'
|
|
132
|
+
|
|
133
|
+
- type: website
|
|
134
|
+
product_name: 'gloo-mesh-core'
|
|
135
|
+
version: 'latest'
|
|
136
|
+
url: 'https://docs.solo.io/gloo-mesh/main/'
|
|
137
|
+
max_size: 1048576
|
|
138
|
+
database_config:
|
|
139
|
+
type: 'qdrant'
|
|
140
|
+
params:
|
|
141
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
142
|
+
qdrant_port: 443
|
|
143
|
+
collection_name: 'gloo-mesh-core'
|
|
144
|
+
|
|
145
|
+
- type: website
|
|
146
|
+
product_name: 'gloo-mesh-enterprise'
|
|
147
|
+
version: 'latest'
|
|
148
|
+
url: 'https://docs.solo.io/gloo-mesh-enterprise/main/'
|
|
149
|
+
max_size: 1048576
|
|
150
|
+
database_config:
|
|
151
|
+
type: 'qdrant'
|
|
152
|
+
params:
|
|
153
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
154
|
+
qdrant_port: 443
|
|
155
|
+
collection_name: 'gloo-mesh-enterprise'
|
|
156
|
+
|
|
157
|
+
- type: website
|
|
158
|
+
product_name: 'gloo-edge'
|
|
159
|
+
version: 'latest'
|
|
160
|
+
url: 'https://docs.solo.io/gloo-edge/main/'
|
|
161
|
+
max_size: 1048576
|
|
162
|
+
database_config:
|
|
163
|
+
type: 'qdrant'
|
|
164
|
+
params:
|
|
165
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
166
|
+
qdrant_port: 443
|
|
167
|
+
collection_name: 'gloo-edge'
|
|
168
|
+
|
|
169
|
+
- type: website
|
|
170
|
+
product_name: 'gloo-gateway'
|
|
171
|
+
version: 'latest'
|
|
172
|
+
url: 'https://docs.solo.io/gateway/main/'
|
|
173
|
+
max_size: 1048576
|
|
174
|
+
database_config:
|
|
175
|
+
type: 'qdrant'
|
|
176
|
+
params:
|
|
177
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
178
|
+
qdrant_port: 443
|
|
179
|
+
collection_name: 'gloo-gateway'
|
|
180
|
+
|
|
181
|
+
- type: website
|
|
182
|
+
product_name: 'helm'
|
|
183
|
+
version: 'latest'
|
|
184
|
+
url: 'https://helm.sh/docs/'
|
|
185
|
+
max_size: 1048576
|
|
186
|
+
database_config:
|
|
187
|
+
type: 'qdrant'
|
|
188
|
+
params:
|
|
189
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
190
|
+
qdrant_port: 443
|
|
191
|
+
collection_name: 'helm'
|
|
192
|
+
|
|
193
|
+
- type: website
|
|
194
|
+
product_name: 'istio'
|
|
195
|
+
version: 'latest'
|
|
196
|
+
url: 'https://istio.io/latest/docs/'
|
|
197
|
+
max_size: 1048576
|
|
198
|
+
database_config:
|
|
199
|
+
type: 'qdrant'
|
|
200
|
+
params:
|
|
201
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
202
|
+
qdrant_port: 443
|
|
203
|
+
collection_name: 'istio'
|
|
204
|
+
|
|
205
|
+
- type: website
|
|
206
|
+
product_name: 'kgateway'
|
|
207
|
+
version: 'latest'
|
|
208
|
+
url: 'https://kgateway.dev/docs/'
|
|
209
|
+
max_size: 1048576
|
|
210
|
+
database_config:
|
|
211
|
+
type: 'qdrant'
|
|
212
|
+
params:
|
|
213
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
214
|
+
qdrant_port: 443
|
|
215
|
+
collection_name: 'kgateway'
|
|
216
|
+
|
|
217
|
+
- type: website
|
|
218
|
+
product_name: 'kubernetes'
|
|
219
|
+
version: 'latest'
|
|
220
|
+
url: 'https://kubernetes.io/docs/'
|
|
221
|
+
max_size: 1048576
|
|
222
|
+
database_config:
|
|
223
|
+
type: 'qdrant'
|
|
224
|
+
params:
|
|
225
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
226
|
+
qdrant_port: 443
|
|
227
|
+
collection_name: 'kubernetes'
|
|
228
|
+
|
|
229
|
+
- type: website
|
|
230
|
+
product_name: 'otel'
|
|
231
|
+
version: 'latest'
|
|
232
|
+
url: 'https://opentelemetry.io/docs/'
|
|
233
|
+
max_size: 1048576
|
|
234
|
+
database_config:
|
|
235
|
+
type: 'qdrant'
|
|
236
|
+
params:
|
|
237
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
238
|
+
qdrant_port: 443
|
|
239
|
+
collection_name: 'otel'
|
|
240
|
+
|
|
241
|
+
- type: website
|
|
242
|
+
product_name: 'prometheus'
|
|
243
|
+
version: 'latest'
|
|
244
|
+
url: 'https://prometheus.io/docs/'
|
|
245
|
+
max_size: 1048576
|
|
246
|
+
database_config:
|
|
247
|
+
type: 'qdrant'
|
|
248
|
+
params:
|
|
249
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
250
|
+
qdrant_port: 443
|
|
251
|
+
collection_name: 'prometheus'
|
package/dist/doc2vec.js
CHANGED
|
@@ -593,8 +593,31 @@ class Doc2Vec {
|
|
|
593
593
|
processedFiles.add(filePath);
|
|
594
594
|
logger.info(`Processing content from ${filePath} (${content.length} chars)`);
|
|
595
595
|
try {
|
|
596
|
-
//
|
|
597
|
-
|
|
596
|
+
// Generate URL based on configuration
|
|
597
|
+
let fileUrl;
|
|
598
|
+
if (config.url_rewrite_prefix) {
|
|
599
|
+
// Replace local path with URL prefix
|
|
600
|
+
const relativePath = path.relative(config.path, filePath).replace(/\\/g, '/');
|
|
601
|
+
// If relativePath starts with '..', it means the file is outside the base directory
|
|
602
|
+
if (relativePath.startsWith('..')) {
|
|
603
|
+
// For files outside the configured path, use the default file:// scheme
|
|
604
|
+
fileUrl = `file://${filePath}`;
|
|
605
|
+
logger.debug(`File outside configured path, using default URL: ${fileUrl}`);
|
|
606
|
+
}
|
|
607
|
+
else {
|
|
608
|
+
// For files inside the configured path, rewrite the URL
|
|
609
|
+
// Handle trailing slashes in the URL prefix to avoid double slashes
|
|
610
|
+
const prefix = config.url_rewrite_prefix.endsWith('/')
|
|
611
|
+
? config.url_rewrite_prefix.slice(0, -1)
|
|
612
|
+
: config.url_rewrite_prefix;
|
|
613
|
+
fileUrl = `${prefix}/${relativePath}`;
|
|
614
|
+
logger.debug(`URL rewritten: ${filePath} -> ${fileUrl}`);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
else {
|
|
618
|
+
// Use default file:// URL
|
|
619
|
+
fileUrl = `file://${filePath}`;
|
|
620
|
+
}
|
|
598
621
|
const chunks = await this.chunkMarkdown(content, config, fileUrl);
|
|
599
622
|
logger.info(`Created ${chunks.length} chunks`);
|
|
600
623
|
if (chunks.length > 0) {
|
|
@@ -667,15 +690,14 @@ class Doc2Vec {
|
|
|
667
690
|
logger.error(`Error during chunking or embedding for ${filePath}:`, error);
|
|
668
691
|
}
|
|
669
692
|
}, logger);
|
|
670
|
-
logger.info(`Found ${validChunkIds.size} valid chunks across processed files for ${config.path}`);
|
|
671
693
|
logger.section('CLEANUP');
|
|
672
694
|
if (dbConnection.type === 'sqlite') {
|
|
673
695
|
logger.info(`Running SQLite cleanup for local directory ${config.path}`);
|
|
674
|
-
this.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config
|
|
696
|
+
this.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
|
|
675
697
|
}
|
|
676
698
|
else if (dbConnection.type === 'qdrant') {
|
|
677
699
|
logger.info(`Running Qdrant cleanup for local directory ${config.path} in collection ${dbConnection.collectionName}`);
|
|
678
|
-
await this.removeObsoleteFilesQdrant(dbConnection, processedFiles, config
|
|
700
|
+
await this.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
|
|
679
701
|
}
|
|
680
702
|
logger.info(`Finished processing local directory: ${config.path}`);
|
|
681
703
|
}
|
|
@@ -765,6 +787,139 @@ class Doc2Vec {
|
|
|
765
787
|
logger.error(`Error reading directory ${dirPath}:`, error);
|
|
766
788
|
}
|
|
767
789
|
}
|
|
790
|
+
removeObsoleteFilesSQLite(db, processedFiles, pathConfig, logger) {
|
|
791
|
+
const getChunksForPathStmt = db.prepare(`
|
|
792
|
+
SELECT chunk_id, url FROM vec_items
|
|
793
|
+
WHERE url LIKE ? || '%'
|
|
794
|
+
`);
|
|
795
|
+
const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
|
|
796
|
+
// Determine if we're using URL rewriting or direct file paths
|
|
797
|
+
const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
|
|
798
|
+
// Set up the URL prefix for searching
|
|
799
|
+
let urlPrefix;
|
|
800
|
+
if (isRewriteMode) {
|
|
801
|
+
// Handle URL rewriting case
|
|
802
|
+
urlPrefix = pathConfig.url_rewrite_prefix || '';
|
|
803
|
+
urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
|
|
804
|
+
}
|
|
805
|
+
else {
|
|
806
|
+
// Handle direct file path case
|
|
807
|
+
const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
|
|
808
|
+
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
809
|
+
urlPrefix = `file://${cleanedDirPrefix}`;
|
|
810
|
+
}
|
|
811
|
+
logger.debug(`Searching for chunks with URL prefix: ${urlPrefix}`);
|
|
812
|
+
const existingChunks = getChunksForPathStmt.all(urlPrefix);
|
|
813
|
+
let deletedCount = 0;
|
|
814
|
+
const transaction = db.transaction(() => {
|
|
815
|
+
for (const { chunk_id, url } of existingChunks) {
|
|
816
|
+
// Skip if it's not from our URL prefix (safety check)
|
|
817
|
+
if (!url.startsWith(urlPrefix))
|
|
818
|
+
continue;
|
|
819
|
+
let filePath;
|
|
820
|
+
let shouldDelete = false;
|
|
821
|
+
if (isRewriteMode) {
|
|
822
|
+
// URL rewrite mode: extract relative path and construct full file path
|
|
823
|
+
const config = pathConfig;
|
|
824
|
+
const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
|
|
825
|
+
filePath = path.join(config.path, relativePath);
|
|
826
|
+
shouldDelete = !processedFiles.has(filePath);
|
|
827
|
+
}
|
|
828
|
+
else {
|
|
829
|
+
// Direct file path mode: remove file:// prefix to match with processedFiles
|
|
830
|
+
filePath = url.substring(7); // Remove 'file://' prefix
|
|
831
|
+
shouldDelete = !processedFiles.has(filePath);
|
|
832
|
+
}
|
|
833
|
+
if (shouldDelete) {
|
|
834
|
+
logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed: ${filePath})`);
|
|
835
|
+
deleteChunkStmt.run(chunk_id);
|
|
836
|
+
deletedCount++;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
});
|
|
840
|
+
transaction();
|
|
841
|
+
logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL prefix ${urlPrefix}`);
|
|
842
|
+
}
|
|
843
|
+
async removeObsoleteFilesQdrant(db, processedFiles, pathConfig, logger) {
|
|
844
|
+
const { client, collectionName } = db;
|
|
845
|
+
try {
|
|
846
|
+
// Determine if we're using URL rewriting or direct file paths
|
|
847
|
+
const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
|
|
848
|
+
// Set up the URL prefix for searching
|
|
849
|
+
let urlPrefix;
|
|
850
|
+
if (isRewriteMode) {
|
|
851
|
+
// Handle URL rewriting case
|
|
852
|
+
urlPrefix = pathConfig.url_rewrite_prefix || '';
|
|
853
|
+
urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
|
|
854
|
+
}
|
|
855
|
+
else {
|
|
856
|
+
// Handle direct file path case
|
|
857
|
+
const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
|
|
858
|
+
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
859
|
+
urlPrefix = `file://${cleanedDirPrefix}`;
|
|
860
|
+
}
|
|
861
|
+
logger.debug(`Checking for obsolete chunks with URL prefix: ${urlPrefix}`);
|
|
862
|
+
const response = await client.scroll(collectionName, {
|
|
863
|
+
limit: 10000,
|
|
864
|
+
with_payload: true,
|
|
865
|
+
with_vector: false,
|
|
866
|
+
filter: {
|
|
867
|
+
must: [
|
|
868
|
+
{
|
|
869
|
+
key: "url",
|
|
870
|
+
match: {
|
|
871
|
+
text: urlPrefix
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
],
|
|
875
|
+
must_not: [
|
|
876
|
+
{
|
|
877
|
+
key: "is_metadata",
|
|
878
|
+
match: {
|
|
879
|
+
value: true
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
]
|
|
883
|
+
}
|
|
884
|
+
});
|
|
885
|
+
const obsoletePointIds = response.points
|
|
886
|
+
.filter((point) => {
|
|
887
|
+
const url = point.payload?.url;
|
|
888
|
+
// Double check it's not a metadata record
|
|
889
|
+
if (point.payload?.is_metadata === true) {
|
|
890
|
+
return false;
|
|
891
|
+
}
|
|
892
|
+
if (!url || !url.startsWith(urlPrefix)) {
|
|
893
|
+
return false;
|
|
894
|
+
}
|
|
895
|
+
let filePath;
|
|
896
|
+
if (isRewriteMode) {
|
|
897
|
+
// URL rewrite mode: extract relative path and construct full file path
|
|
898
|
+
const config = pathConfig;
|
|
899
|
+
const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
|
|
900
|
+
filePath = path.join(config.path, relativePath);
|
|
901
|
+
}
|
|
902
|
+
else {
|
|
903
|
+
// Direct file path mode: remove file:// prefix to match with processedFiles
|
|
904
|
+
filePath = url.startsWith('file://') ? url.substring(7) : '';
|
|
905
|
+
}
|
|
906
|
+
return filePath && !processedFiles.has(filePath);
|
|
907
|
+
})
|
|
908
|
+
.map((point) => point.id);
|
|
909
|
+
if (obsoletePointIds.length > 0) {
|
|
910
|
+
await client.delete(collectionName, {
|
|
911
|
+
points: obsoletePointIds,
|
|
912
|
+
});
|
|
913
|
+
logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for URL prefix ${urlPrefix}`);
|
|
914
|
+
}
|
|
915
|
+
else {
|
|
916
|
+
logger.info(`No obsolete chunks to delete from Qdrant for URL prefix ${urlPrefix}`);
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
catch (error) {
|
|
920
|
+
logger.error(`Error removing obsolete chunks from Qdrant:`, error);
|
|
921
|
+
}
|
|
922
|
+
}
|
|
768
923
|
async initDatabase(config, parentLogger) {
|
|
769
924
|
const logger = parentLogger.child('database');
|
|
770
925
|
const dbConfig = config.database_config;
|
|
@@ -938,85 +1093,6 @@ class Doc2Vec {
|
|
|
938
1093
|
transaction();
|
|
939
1094
|
logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL ${urlPrefix}`);
|
|
940
1095
|
}
|
|
941
|
-
removeObsoleteFilesSQLite(db, processedFiles, dirPrefix, logger) {
|
|
942
|
-
const getChunksForDirStmt = db.prepare(`
|
|
943
|
-
SELECT chunk_id, url FROM vec_items
|
|
944
|
-
WHERE url LIKE 'file://%' AND url LIKE ?
|
|
945
|
-
`);
|
|
946
|
-
const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
|
|
947
|
-
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
948
|
-
const filePrefix = `file://${cleanedDirPrefix}`;
|
|
949
|
-
const existingChunks = getChunksForDirStmt.all(`${filePrefix}%`);
|
|
950
|
-
let deletedCount = 0;
|
|
951
|
-
const transaction = db.transaction(() => {
|
|
952
|
-
for (const { chunk_id, url } of existingChunks) {
|
|
953
|
-
// Remove file:// prefix to match with processedFiles
|
|
954
|
-
const filePath = url.substring(7);
|
|
955
|
-
if (!processedFiles.has(filePath)) {
|
|
956
|
-
logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed)`);
|
|
957
|
-
deleteChunkStmt.run(chunk_id);
|
|
958
|
-
deletedCount++;
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
});
|
|
962
|
-
transaction();
|
|
963
|
-
logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for directory ${dirPrefix}`);
|
|
964
|
-
}
|
|
965
|
-
async removeObsoleteFilesQdrant(db, processedFiles, dirPrefix, logger) {
|
|
966
|
-
const { client, collectionName } = db;
|
|
967
|
-
try {
|
|
968
|
-
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
969
|
-
const filePrefix = `file://${cleanedDirPrefix}`;
|
|
970
|
-
// Get all points that match the file prefix but are not metadata points
|
|
971
|
-
const response = await client.scroll(collectionName, {
|
|
972
|
-
limit: 10000,
|
|
973
|
-
with_payload: true,
|
|
974
|
-
with_vector: false,
|
|
975
|
-
filter: {
|
|
976
|
-
must: [
|
|
977
|
-
{
|
|
978
|
-
key: "url",
|
|
979
|
-
match: {
|
|
980
|
-
text: `${filePrefix}`
|
|
981
|
-
}
|
|
982
|
-
}
|
|
983
|
-
],
|
|
984
|
-
must_not: [
|
|
985
|
-
{
|
|
986
|
-
key: "is_metadata",
|
|
987
|
-
match: {
|
|
988
|
-
value: true
|
|
989
|
-
}
|
|
990
|
-
}
|
|
991
|
-
]
|
|
992
|
-
}
|
|
993
|
-
});
|
|
994
|
-
const obsoletePointIds = response.points
|
|
995
|
-
.filter((point) => {
|
|
996
|
-
const url = point.payload?.url;
|
|
997
|
-
// Double check it's not a metadata record
|
|
998
|
-
if (point.payload?.is_metadata === true) {
|
|
999
|
-
return false;
|
|
1000
|
-
}
|
|
1001
|
-
// Remove file:// prefix to match with processedFiles
|
|
1002
|
-
const filePath = url && url.startsWith('file://') ? url.substring(7) : '';
|
|
1003
|
-
return filePath && !processedFiles.has(filePath);
|
|
1004
|
-
})
|
|
1005
|
-
.map((point) => point.id);
|
|
1006
|
-
if (obsoletePointIds.length > 0) {
|
|
1007
|
-
await client.delete(collectionName, {
|
|
1008
|
-
points: obsoletePointIds,
|
|
1009
|
-
});
|
|
1010
|
-
logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for directory ${dirPrefix}`);
|
|
1011
|
-
}
|
|
1012
|
-
else {
|
|
1013
|
-
logger.info(`No obsolete chunks to delete from Qdrant for directory ${dirPrefix}`);
|
|
1014
|
-
}
|
|
1015
|
-
}
|
|
1016
|
-
catch (error) {
|
|
1017
|
-
logger.error(`Error removing obsolete chunks from Qdrant:`, error);
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
1096
|
isValidUuid(str) {
|
|
1021
1097
|
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
1022
1098
|
return uuidRegex.test(str);
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ACMCdkStack = void 0;
|
|
4
|
+
const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
5
|
+
class ACMCdkStack extends aws_cdk_lib_1.Stack {
|
|
6
|
+
constructor(scope, id, props) {
|
|
7
|
+
super(scope, id, props);
|
|
8
|
+
if (!process.env.DOMAIN_NAME) {
|
|
9
|
+
throw new Error("DOMAIN_NAME environment variable is not defined");
|
|
10
|
+
}
|
|
11
|
+
const hostedZone = props.hostedZone;
|
|
12
|
+
const edgeSiteDomainName = "apps." + process.env.DOMAIN_NAME;
|
|
13
|
+
this.acmCertificate = new aws_cdk_lib_1.aws_certificatemanager.Certificate(this, 'SiteCertificate', {
|
|
14
|
+
domainName: edgeSiteDomainName,
|
|
15
|
+
validation: aws_cdk_lib_1.aws_certificatemanager.CertificateValidation.fromDns(hostedZone),
|
|
16
|
+
});
|
|
17
|
+
new aws_cdk_lib_1.CfnOutput(this, 'ACMCertificate', { value: this.acmCertificate.certificateArn });
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
exports.ACMCdkStack = ACMCdkStack;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Route53CdkStack = void 0;
|
|
4
|
+
const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
5
|
+
class Route53CdkStack extends aws_cdk_lib_1.Stack {
|
|
6
|
+
constructor(scope, id, props) {
|
|
7
|
+
super(scope, id, props);
|
|
8
|
+
if (!process.env.PARENT_DOMAIN_NAME) {
|
|
9
|
+
throw new Error("PARENT_DOMAIN_NAME environment variable is not defined");
|
|
10
|
+
}
|
|
11
|
+
if (!process.env.DOMAIN_NAME) {
|
|
12
|
+
throw new Error("DOMAIN_NAME environment variable is not defined");
|
|
13
|
+
}
|
|
14
|
+
this.hostedZone = new aws_cdk_lib_1.aws_route53.PublicHostedZone(this, "HostedZone", {
|
|
15
|
+
zoneName: process.env.DOMAIN_NAME,
|
|
16
|
+
comment: "Managed by CDK"
|
|
17
|
+
});
|
|
18
|
+
const parent = aws_cdk_lib_1.aws_route53.HostedZone.fromLookup(this, "ParentHostedZone", {
|
|
19
|
+
domainName: process.env.PARENT_DOMAIN_NAME + "."
|
|
20
|
+
});
|
|
21
|
+
if (!this.hostedZone.hostedZoneNameServers) {
|
|
22
|
+
throw new Error("NS record is undefined.");
|
|
23
|
+
}
|
|
24
|
+
new aws_cdk_lib_1.aws_route53.ZoneDelegationRecord(this, "NS", {
|
|
25
|
+
zone: parent,
|
|
26
|
+
recordName: process.env.DOMAIN_NAME,
|
|
27
|
+
nameServers: this.hostedZone.hostedZoneNameServers,
|
|
28
|
+
comment: "Managed by CDK"
|
|
29
|
+
});
|
|
30
|
+
new aws_cdk_lib_1.CfnOutput(this, 'ZoneId', { value: this.hostedZone.hostedZoneId });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
exports.Route53CdkStack = Route53CdkStack;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.WAFCdkStack = void 0;
|
|
4
|
+
const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
5
|
+
class WAFCdkStack extends aws_cdk_lib_1.Stack {
|
|
6
|
+
constructor(scope, id, props) {
|
|
7
|
+
super(scope, id, props);
|
|
8
|
+
const webAcl = new aws_cdk_lib_1.aws_wafv2.CfnWebACL(this, 'RateLimitWebAcl', {
|
|
9
|
+
description: 'WebACL to rate limit the calls',
|
|
10
|
+
defaultAction: { allow: {} },
|
|
11
|
+
scope: 'REGIONAL',
|
|
12
|
+
visibilityConfig: {
|
|
13
|
+
metricName: 'RateLimitWebAclMetric',
|
|
14
|
+
cloudWatchMetricsEnabled: true,
|
|
15
|
+
sampledRequestsEnabled: true,
|
|
16
|
+
},
|
|
17
|
+
rules: [
|
|
18
|
+
{
|
|
19
|
+
// rate limit to 100 based on IP within 5 minutes
|
|
20
|
+
name: 'AWSRateLimitRuleSet',
|
|
21
|
+
priority: 1,
|
|
22
|
+
action: { block: {} },
|
|
23
|
+
visibilityConfig: {
|
|
24
|
+
metricName: "AWSRateLimitIP",
|
|
25
|
+
cloudWatchMetricsEnabled: true,
|
|
26
|
+
sampledRequestsEnabled: false
|
|
27
|
+
},
|
|
28
|
+
statement: {
|
|
29
|
+
rateBasedStatement: {
|
|
30
|
+
aggregateKeyType: "IP",
|
|
31
|
+
limit: 100,
|
|
32
|
+
evaluationWindowSec: 300
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
// common rule set
|
|
38
|
+
name: 'AWSManagedRulesCommonRuleSet',
|
|
39
|
+
priority: 2,
|
|
40
|
+
overrideAction: { none: {} },
|
|
41
|
+
visibilityConfig: {
|
|
42
|
+
metricName: 'AWSManagedRulesCommonRuleSetMetric',
|
|
43
|
+
cloudWatchMetricsEnabled: true,
|
|
44
|
+
sampledRequestsEnabled: true
|
|
45
|
+
},
|
|
46
|
+
statement: {
|
|
47
|
+
managedRuleGroupStatement: {
|
|
48
|
+
vendorName: 'AWS',
|
|
49
|
+
name: 'AWSManagedRulesCommonRuleSet'
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
]
|
|
54
|
+
});
|
|
55
|
+
const alb = aws_cdk_lib_1.aws_elasticloadbalancingv2.ApplicationLoadBalancer.fromLookup(this, 'ALBListener', {
|
|
56
|
+
loadBalancerTags: {
|
|
57
|
+
'elbv2.k8s.aws/cluster': process.env.CLUSTER_OWNER + '-' + process.env.PROJECT
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
new aws_cdk_lib_1.aws_wafv2.CfnWebACLAssociation(this, "ALBWebACLAssociation", {
|
|
61
|
+
webAclArn: webAcl.attrArn,
|
|
62
|
+
resourceArn: alb.loadBalancerArn
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
exports.WAFCdkStack = WAFCdkStack;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// import * as cdk from 'aws-cdk-lib';
|
|
3
|
+
// import { Template } from 'aws-cdk-lib/assertions';
|
|
4
|
+
// import * as S3CloudfrontCdk from '../lib/s3-cloudfront-cdk-stack';
|
|
5
|
+
// example test. To run these tests, uncomment this file along with the
|
|
6
|
+
// example resource in lib/s3-cloudfront-cdk-stack.ts
|
|
7
|
+
test('SQS Queue Created', () => {
|
|
8
|
+
// const app = new cdk.App();
|
|
9
|
+
// // WHEN
|
|
10
|
+
// const stack = new S3CloudfrontCdk.S3CloudfrontCdkStack(app, 'MyTestStack');
|
|
11
|
+
// // THEN
|
|
12
|
+
// const template = Template.fromStack(stack);
|
|
13
|
+
// template.hasResourceProperties('AWS::SQS::Queue', {
|
|
14
|
+
// VisibilityTimeout: 300
|
|
15
|
+
// });
|
|
16
|
+
});
|