doc2vec 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/config.yaml.new +239 -165
- package/config.yaml.ok +164 -0
- package/config.yaml.solo +251 -0
- package/dist/doc2vec.js +164 -84
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/lib/acm-cdk-stack.js +20 -0
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/lib/route53-cdk-stack.js +33 -0
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/lib/waf-cdk-stack.js +66 -0
- package/dist/solo-reference-architectures/Gloo_Edge/ALB_Last_Mile_Encryption/data/acm-waf-cdk/test/s3-cloudfront-cdk.test.js +16 -0
- package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/lib/acm-cdk-stack.js +21 -0
- package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/lib/s3-cloudfront-cdk-stack.js +95 -0
- package/dist/solo-reference-architectures/Gloo_Edge/Cloudfront_NLB/data/s3-cloudfront-cdk/test/s3-cloudfront-cdk.test.js +16 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/bin/acm-cdk.js +55 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/lib/acm-cdk-stack.js +20 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/lib/route53-cdk-stack.js +33 -0
- package/dist/solo-reference-architectures/Gloo_Gateway/aws-nlb-tls-offloading/data/acm-cdk/test/s3-cloudfront-cdk.test.js +16 -0
- package/doc2vec.ts +184 -93
- package/doc2vec.ts.ok +1730 -0
- package/package.json +1 -1
- package/tmp/README.md.old +1 -0
- package/tmp.db +0 -0
package/config.yaml.solo
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
# Doc2Vec Configuration
|
|
2
|
+
sources:
|
|
3
|
+
# Local directory Sources
|
|
4
|
+
- type: local_directory
|
|
5
|
+
product_name: 'solo-reference-architectures'
|
|
6
|
+
version: 'latest'
|
|
7
|
+
path: './solo-reference-architectures'
|
|
8
|
+
include_extensions: ['.md']
|
|
9
|
+
exclude_extensions: []
|
|
10
|
+
recursive: true
|
|
11
|
+
url_rewrite_prefix: https://github.com/solo-io/solo-reference-architectures/blob/main
|
|
12
|
+
max_size: 1048576
|
|
13
|
+
database_config:
|
|
14
|
+
type: 'qdrant'
|
|
15
|
+
params:
|
|
16
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
17
|
+
qdrant_port: 443
|
|
18
|
+
collection_name: 'github-solo-reference-architectures'
|
|
19
|
+
|
|
20
|
+
# GitHub Sources
|
|
21
|
+
- type: github
|
|
22
|
+
product_name: 'istio'
|
|
23
|
+
version: 'latest'
|
|
24
|
+
repo: 'istio/istio'
|
|
25
|
+
start_date: '2024-01-01'
|
|
26
|
+
max_size: 1048576
|
|
27
|
+
database_config:
|
|
28
|
+
type: 'qdrant'
|
|
29
|
+
params:
|
|
30
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
31
|
+
qdrant_port: 443
|
|
32
|
+
collection_name: 'github-istio'
|
|
33
|
+
|
|
34
|
+
- type: github
|
|
35
|
+
product_name: 'solo-projects'
|
|
36
|
+
version: 'latest'
|
|
37
|
+
repo: 'solo-io/solo-projects'
|
|
38
|
+
start_date: '2024-01-01'
|
|
39
|
+
max_size: 1048576
|
|
40
|
+
database_config:
|
|
41
|
+
type: 'qdrant'
|
|
42
|
+
params:
|
|
43
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
44
|
+
qdrant_port: 443
|
|
45
|
+
collection_name: 'github-solo-projects'
|
|
46
|
+
|
|
47
|
+
- type: github
|
|
48
|
+
product_name: 'gloo-mesh-enterprise'
|
|
49
|
+
version: 'latest'
|
|
50
|
+
repo: 'solo-io/gloo-mesh-enterprise'
|
|
51
|
+
start_date: '2024-01-01'
|
|
52
|
+
max_size: 1048576
|
|
53
|
+
database_config:
|
|
54
|
+
type: 'qdrant'
|
|
55
|
+
params:
|
|
56
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
57
|
+
qdrant_port: 443
|
|
58
|
+
collection_name: 'github-gloo-mesh-enterprise'
|
|
59
|
+
|
|
60
|
+
# Website Sources
|
|
61
|
+
- type: website
|
|
62
|
+
product_name: 'ambient'
|
|
63
|
+
version: 'latest'
|
|
64
|
+
url: 'https://ambientmesh.io/docs/'
|
|
65
|
+
max_size: 1048576
|
|
66
|
+
database_config:
|
|
67
|
+
type: 'qdrant'
|
|
68
|
+
params:
|
|
69
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
70
|
+
qdrant_port: 443
|
|
71
|
+
collection_name: 'ambient'
|
|
72
|
+
|
|
73
|
+
- type: website
|
|
74
|
+
product_name: 'argo'
|
|
75
|
+
version: 'latest'
|
|
76
|
+
url: 'https://argo-cd.readthedocs.io/en/stable/'
|
|
77
|
+
max_size: 1048576
|
|
78
|
+
database_config:
|
|
79
|
+
type: 'qdrant'
|
|
80
|
+
params:
|
|
81
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
82
|
+
qdrant_port: 443
|
|
83
|
+
collection_name: 'argo'
|
|
84
|
+
|
|
85
|
+
- type: website
|
|
86
|
+
product_name: 'argo-rollouts'
|
|
87
|
+
version: 'latest'
|
|
88
|
+
url: 'https://argoproj.github.io/argo-rollouts/installation/'
|
|
89
|
+
max_size: 1048576
|
|
90
|
+
database_config:
|
|
91
|
+
type: 'qdrant'
|
|
92
|
+
params:
|
|
93
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
94
|
+
qdrant_port: 443
|
|
95
|
+
collection_name: 'argo-rollouts'
|
|
96
|
+
|
|
97
|
+
- type: website
|
|
98
|
+
product_name: 'argo-rollouts'
|
|
99
|
+
version: 'latest'
|
|
100
|
+
url: 'https://rollouts-plugin-trafficrouter-gatewayapi.readthedocs.io/en/latest/'
|
|
101
|
+
max_size: 1048576
|
|
102
|
+
database_config:
|
|
103
|
+
type: 'qdrant'
|
|
104
|
+
params:
|
|
105
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
106
|
+
qdrant_port: 443
|
|
107
|
+
collection_name: 'argo-rollouts'
|
|
108
|
+
|
|
109
|
+
- type: website
|
|
110
|
+
product_name: 'cilium'
|
|
111
|
+
version: 'latest'
|
|
112
|
+
url: 'https://docs.cilium.io/en/stable/'
|
|
113
|
+
max_size: 1048576
|
|
114
|
+
database_config:
|
|
115
|
+
type: 'qdrant'
|
|
116
|
+
params:
|
|
117
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
118
|
+
qdrant_port: 443
|
|
119
|
+
collection_name: 'cilium'
|
|
120
|
+
|
|
121
|
+
- type: website
|
|
122
|
+
product_name: 'gateway-api'
|
|
123
|
+
version: 'latest'
|
|
124
|
+
url: 'https://gateway-api.sigs.k8s.io/'
|
|
125
|
+
max_size: 1048576
|
|
126
|
+
database_config:
|
|
127
|
+
type: 'qdrant'
|
|
128
|
+
params:
|
|
129
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
130
|
+
qdrant_port: 443
|
|
131
|
+
collection_name: 'gateway-api'
|
|
132
|
+
|
|
133
|
+
- type: website
|
|
134
|
+
product_name: 'gloo-mesh-core'
|
|
135
|
+
version: 'latest'
|
|
136
|
+
url: 'https://docs.solo.io/gloo-mesh/main/'
|
|
137
|
+
max_size: 1048576
|
|
138
|
+
database_config:
|
|
139
|
+
type: 'qdrant'
|
|
140
|
+
params:
|
|
141
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
142
|
+
qdrant_port: 443
|
|
143
|
+
collection_name: 'gloo-mesh-core'
|
|
144
|
+
|
|
145
|
+
- type: website
|
|
146
|
+
product_name: 'gloo-mesh-enterprise'
|
|
147
|
+
version: 'latest'
|
|
148
|
+
url: 'https://docs.solo.io/gloo-mesh-enterprise/main/'
|
|
149
|
+
max_size: 1048576
|
|
150
|
+
database_config:
|
|
151
|
+
type: 'qdrant'
|
|
152
|
+
params:
|
|
153
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
154
|
+
qdrant_port: 443
|
|
155
|
+
collection_name: 'gloo-mesh-enterprise'
|
|
156
|
+
|
|
157
|
+
- type: website
|
|
158
|
+
product_name: 'gloo-edge'
|
|
159
|
+
version: 'latest'
|
|
160
|
+
url: 'https://docs.solo.io/gloo-edge/main/'
|
|
161
|
+
max_size: 1048576
|
|
162
|
+
database_config:
|
|
163
|
+
type: 'qdrant'
|
|
164
|
+
params:
|
|
165
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
166
|
+
qdrant_port: 443
|
|
167
|
+
collection_name: 'gloo-edge'
|
|
168
|
+
|
|
169
|
+
- type: website
|
|
170
|
+
product_name: 'gloo-gateway'
|
|
171
|
+
version: 'latest'
|
|
172
|
+
url: 'https://docs.solo.io/gateway/main/'
|
|
173
|
+
max_size: 1048576
|
|
174
|
+
database_config:
|
|
175
|
+
type: 'qdrant'
|
|
176
|
+
params:
|
|
177
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
178
|
+
qdrant_port: 443
|
|
179
|
+
collection_name: 'gloo-gateway'
|
|
180
|
+
|
|
181
|
+
- type: website
|
|
182
|
+
product_name: 'helm'
|
|
183
|
+
version: 'latest'
|
|
184
|
+
url: 'https://helm.sh/docs/'
|
|
185
|
+
max_size: 1048576
|
|
186
|
+
database_config:
|
|
187
|
+
type: 'qdrant'
|
|
188
|
+
params:
|
|
189
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
190
|
+
qdrant_port: 443
|
|
191
|
+
collection_name: 'helm'
|
|
192
|
+
|
|
193
|
+
- type: website
|
|
194
|
+
product_name: 'istio'
|
|
195
|
+
version: 'latest'
|
|
196
|
+
url: 'https://istio.io/latest/docs/'
|
|
197
|
+
max_size: 1048576
|
|
198
|
+
database_config:
|
|
199
|
+
type: 'qdrant'
|
|
200
|
+
params:
|
|
201
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
202
|
+
qdrant_port: 443
|
|
203
|
+
collection_name: 'istio'
|
|
204
|
+
|
|
205
|
+
- type: website
|
|
206
|
+
product_name: 'kgateway'
|
|
207
|
+
version: 'latest'
|
|
208
|
+
url: 'https://kgateway.dev/docs/'
|
|
209
|
+
max_size: 1048576
|
|
210
|
+
database_config:
|
|
211
|
+
type: 'qdrant'
|
|
212
|
+
params:
|
|
213
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
214
|
+
qdrant_port: 443
|
|
215
|
+
collection_name: 'kgateway'
|
|
216
|
+
|
|
217
|
+
- type: website
|
|
218
|
+
product_name: 'kubernetes'
|
|
219
|
+
version: 'latest'
|
|
220
|
+
url: 'https://kubernetes.io/docs/'
|
|
221
|
+
max_size: 1048576
|
|
222
|
+
database_config:
|
|
223
|
+
type: 'qdrant'
|
|
224
|
+
params:
|
|
225
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
226
|
+
qdrant_port: 443
|
|
227
|
+
collection_name: 'kubernetes'
|
|
228
|
+
|
|
229
|
+
- type: website
|
|
230
|
+
product_name: 'otel'
|
|
231
|
+
version: 'latest'
|
|
232
|
+
url: 'https://opentelemetry.io/docs/'
|
|
233
|
+
max_size: 1048576
|
|
234
|
+
database_config:
|
|
235
|
+
type: 'qdrant'
|
|
236
|
+
params:
|
|
237
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
238
|
+
qdrant_port: 443
|
|
239
|
+
collection_name: 'otel'
|
|
240
|
+
|
|
241
|
+
- type: website
|
|
242
|
+
product_name: 'prometheus'
|
|
243
|
+
version: 'latest'
|
|
244
|
+
url: 'https://prometheus.io/docs/'
|
|
245
|
+
max_size: 1048576
|
|
246
|
+
database_config:
|
|
247
|
+
type: 'qdrant'
|
|
248
|
+
params:
|
|
249
|
+
qdrant_url: 'https://qdrant.is.solo.io'
|
|
250
|
+
qdrant_port: 443
|
|
251
|
+
collection_name: 'prometheus'
|
package/dist/doc2vec.js
CHANGED
|
@@ -593,8 +593,31 @@ class Doc2Vec {
|
|
|
593
593
|
processedFiles.add(filePath);
|
|
594
594
|
logger.info(`Processing content from ${filePath} (${content.length} chars)`);
|
|
595
595
|
try {
|
|
596
|
-
//
|
|
597
|
-
|
|
596
|
+
// Generate URL based on configuration
|
|
597
|
+
let fileUrl;
|
|
598
|
+
if (config.url_rewrite_prefix) {
|
|
599
|
+
// Replace local path with URL prefix
|
|
600
|
+
const relativePath = path.relative(config.path, filePath).replace(/\\/g, '/');
|
|
601
|
+
// If relativePath starts with '..', it means the file is outside the base directory
|
|
602
|
+
if (relativePath.startsWith('..')) {
|
|
603
|
+
// For files outside the configured path, use the default file:// scheme
|
|
604
|
+
fileUrl = `file://${filePath}`;
|
|
605
|
+
logger.debug(`File outside configured path, using default URL: ${fileUrl}`);
|
|
606
|
+
}
|
|
607
|
+
else {
|
|
608
|
+
// For files inside the configured path, rewrite the URL
|
|
609
|
+
// Handle trailing slashes in the URL prefix to avoid double slashes
|
|
610
|
+
const prefix = config.url_rewrite_prefix.endsWith('/')
|
|
611
|
+
? config.url_rewrite_prefix.slice(0, -1)
|
|
612
|
+
: config.url_rewrite_prefix;
|
|
613
|
+
fileUrl = `${prefix}/${relativePath}`;
|
|
614
|
+
logger.debug(`URL rewritten: ${filePath} -> ${fileUrl}`);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
else {
|
|
618
|
+
// Use default file:// URL
|
|
619
|
+
fileUrl = `file://${filePath}`;
|
|
620
|
+
}
|
|
598
621
|
const chunks = await this.chunkMarkdown(content, config, fileUrl);
|
|
599
622
|
logger.info(`Created ${chunks.length} chunks`);
|
|
600
623
|
if (chunks.length > 0) {
|
|
@@ -667,15 +690,17 @@ class Doc2Vec {
|
|
|
667
690
|
logger.error(`Error during chunking or embedding for ${filePath}:`, error);
|
|
668
691
|
}
|
|
669
692
|
}, logger);
|
|
670
|
-
|
|
693
|
+
// 3. Update cleanup logic for URL rewriting
|
|
671
694
|
logger.section('CLEANUP');
|
|
672
695
|
if (dbConnection.type === 'sqlite') {
|
|
673
696
|
logger.info(`Running SQLite cleanup for local directory ${config.path}`);
|
|
674
|
-
|
|
697
|
+
// Pass the config object directly to our consolidated function
|
|
698
|
+
this.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
|
|
675
699
|
}
|
|
676
700
|
else if (dbConnection.type === 'qdrant') {
|
|
677
701
|
logger.info(`Running Qdrant cleanup for local directory ${config.path} in collection ${dbConnection.collectionName}`);
|
|
678
|
-
|
|
702
|
+
// Pass the config object directly to our consolidated function
|
|
703
|
+
await this.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
|
|
679
704
|
}
|
|
680
705
|
logger.info(`Finished processing local directory: ${config.path}`);
|
|
681
706
|
}
|
|
@@ -765,6 +790,140 @@ class Doc2Vec {
|
|
|
765
790
|
logger.error(`Error reading directory ${dirPath}:`, error);
|
|
766
791
|
}
|
|
767
792
|
}
|
|
793
|
+
// SQLite consolidated function
|
|
794
|
+
removeObsoleteFilesSQLite(db, processedFiles, pathConfig, logger) {
|
|
795
|
+
const getChunksForPathStmt = db.prepare(`
|
|
796
|
+
SELECT chunk_id, url FROM vec_items
|
|
797
|
+
WHERE url LIKE ? || '%'
|
|
798
|
+
`);
|
|
799
|
+
const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
|
|
800
|
+
// Determine if we're using URL rewriting or direct file paths
|
|
801
|
+
const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
|
|
802
|
+
// Set up the URL prefix for searching
|
|
803
|
+
let urlPrefix;
|
|
804
|
+
if (isRewriteMode) {
|
|
805
|
+
// Handle URL rewriting case
|
|
806
|
+
urlPrefix = pathConfig.url_rewrite_prefix || '';
|
|
807
|
+
urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
|
|
808
|
+
}
|
|
809
|
+
else {
|
|
810
|
+
// Handle direct file path case
|
|
811
|
+
const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
|
|
812
|
+
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
813
|
+
urlPrefix = `file://${cleanedDirPrefix}`;
|
|
814
|
+
}
|
|
815
|
+
logger.debug(`Searching for chunks with URL prefix: ${urlPrefix}`);
|
|
816
|
+
const existingChunks = getChunksForPathStmt.all(urlPrefix);
|
|
817
|
+
let deletedCount = 0;
|
|
818
|
+
const transaction = db.transaction(() => {
|
|
819
|
+
for (const { chunk_id, url } of existingChunks) {
|
|
820
|
+
// Skip if it's not from our URL prefix (safety check)
|
|
821
|
+
if (!url.startsWith(urlPrefix))
|
|
822
|
+
continue;
|
|
823
|
+
let filePath;
|
|
824
|
+
let shouldDelete = false;
|
|
825
|
+
if (isRewriteMode) {
|
|
826
|
+
// URL rewrite mode: extract relative path and construct full file path
|
|
827
|
+
const config = pathConfig;
|
|
828
|
+
const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
|
|
829
|
+
filePath = path.join(config.path, relativePath);
|
|
830
|
+
shouldDelete = !processedFiles.has(filePath);
|
|
831
|
+
}
|
|
832
|
+
else {
|
|
833
|
+
// Direct file path mode: remove file:// prefix to match with processedFiles
|
|
834
|
+
filePath = url.substring(7); // Remove 'file://' prefix
|
|
835
|
+
shouldDelete = !processedFiles.has(filePath);
|
|
836
|
+
}
|
|
837
|
+
if (shouldDelete) {
|
|
838
|
+
logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed: ${filePath})`);
|
|
839
|
+
deleteChunkStmt.run(chunk_id);
|
|
840
|
+
deletedCount++;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
});
|
|
844
|
+
transaction();
|
|
845
|
+
logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL prefix ${urlPrefix}`);
|
|
846
|
+
}
|
|
847
|
+
async removeObsoleteFilesQdrant(db, processedFiles, pathConfig, logger) {
|
|
848
|
+
const { client, collectionName } = db;
|
|
849
|
+
try {
|
|
850
|
+
// Determine if we're using URL rewriting or direct file paths
|
|
851
|
+
const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
|
|
852
|
+
// Set up the URL prefix for searching
|
|
853
|
+
let urlPrefix;
|
|
854
|
+
if (isRewriteMode) {
|
|
855
|
+
// Handle URL rewriting case
|
|
856
|
+
urlPrefix = pathConfig.url_rewrite_prefix || '';
|
|
857
|
+
urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
|
|
858
|
+
}
|
|
859
|
+
else {
|
|
860
|
+
// Handle direct file path case
|
|
861
|
+
const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
|
|
862
|
+
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
863
|
+
urlPrefix = `file://${cleanedDirPrefix}`;
|
|
864
|
+
}
|
|
865
|
+
logger.debug(`Checking for obsolete chunks with URL prefix: ${urlPrefix}`);
|
|
866
|
+
const response = await client.scroll(collectionName, {
|
|
867
|
+
limit: 10000,
|
|
868
|
+
with_payload: true,
|
|
869
|
+
with_vector: false,
|
|
870
|
+
filter: {
|
|
871
|
+
must: [
|
|
872
|
+
{
|
|
873
|
+
key: "url",
|
|
874
|
+
match: {
|
|
875
|
+
text: urlPrefix
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
],
|
|
879
|
+
must_not: [
|
|
880
|
+
{
|
|
881
|
+
key: "is_metadata",
|
|
882
|
+
match: {
|
|
883
|
+
value: true
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
]
|
|
887
|
+
}
|
|
888
|
+
});
|
|
889
|
+
const obsoletePointIds = response.points
|
|
890
|
+
.filter((point) => {
|
|
891
|
+
const url = point.payload?.url;
|
|
892
|
+
// Double check it's not a metadata record
|
|
893
|
+
if (point.payload?.is_metadata === true) {
|
|
894
|
+
return false;
|
|
895
|
+
}
|
|
896
|
+
if (!url || !url.startsWith(urlPrefix)) {
|
|
897
|
+
return false;
|
|
898
|
+
}
|
|
899
|
+
let filePath;
|
|
900
|
+
if (isRewriteMode) {
|
|
901
|
+
// URL rewrite mode: extract relative path and construct full file path
|
|
902
|
+
const config = pathConfig;
|
|
903
|
+
const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
|
|
904
|
+
filePath = path.join(config.path, relativePath);
|
|
905
|
+
}
|
|
906
|
+
else {
|
|
907
|
+
// Direct file path mode: remove file:// prefix to match with processedFiles
|
|
908
|
+
filePath = url.startsWith('file://') ? url.substring(7) : '';
|
|
909
|
+
}
|
|
910
|
+
return filePath && !processedFiles.has(filePath);
|
|
911
|
+
})
|
|
912
|
+
.map((point) => point.id);
|
|
913
|
+
if (obsoletePointIds.length > 0) {
|
|
914
|
+
await client.delete(collectionName, {
|
|
915
|
+
points: obsoletePointIds,
|
|
916
|
+
});
|
|
917
|
+
logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for URL prefix ${urlPrefix}`);
|
|
918
|
+
}
|
|
919
|
+
else {
|
|
920
|
+
logger.info(`No obsolete chunks to delete from Qdrant for URL prefix ${urlPrefix}`);
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
catch (error) {
|
|
924
|
+
logger.error(`Error removing obsolete chunks from Qdrant:`, error);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
768
927
|
async initDatabase(config, parentLogger) {
|
|
769
928
|
const logger = parentLogger.child('database');
|
|
770
929
|
const dbConfig = config.database_config;
|
|
@@ -938,85 +1097,6 @@ class Doc2Vec {
|
|
|
938
1097
|
transaction();
|
|
939
1098
|
logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL ${urlPrefix}`);
|
|
940
1099
|
}
|
|
941
|
-
removeObsoleteFilesSQLite(db, processedFiles, dirPrefix, logger) {
|
|
942
|
-
const getChunksForDirStmt = db.prepare(`
|
|
943
|
-
SELECT chunk_id, url FROM vec_items
|
|
944
|
-
WHERE url LIKE 'file://%' AND url LIKE ?
|
|
945
|
-
`);
|
|
946
|
-
const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
|
|
947
|
-
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
948
|
-
const filePrefix = `file://${cleanedDirPrefix}`;
|
|
949
|
-
const existingChunks = getChunksForDirStmt.all(`${filePrefix}%`);
|
|
950
|
-
let deletedCount = 0;
|
|
951
|
-
const transaction = db.transaction(() => {
|
|
952
|
-
for (const { chunk_id, url } of existingChunks) {
|
|
953
|
-
// Remove file:// prefix to match with processedFiles
|
|
954
|
-
const filePath = url.substring(7);
|
|
955
|
-
if (!processedFiles.has(filePath)) {
|
|
956
|
-
logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed)`);
|
|
957
|
-
deleteChunkStmt.run(chunk_id);
|
|
958
|
-
deletedCount++;
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
});
|
|
962
|
-
transaction();
|
|
963
|
-
logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for directory ${dirPrefix}`);
|
|
964
|
-
}
|
|
965
|
-
async removeObsoleteFilesQdrant(db, processedFiles, dirPrefix, logger) {
|
|
966
|
-
const { client, collectionName } = db;
|
|
967
|
-
try {
|
|
968
|
-
const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
|
|
969
|
-
const filePrefix = `file://${cleanedDirPrefix}`;
|
|
970
|
-
// Get all points that match the file prefix but are not metadata points
|
|
971
|
-
const response = await client.scroll(collectionName, {
|
|
972
|
-
limit: 10000,
|
|
973
|
-
with_payload: true,
|
|
974
|
-
with_vector: false,
|
|
975
|
-
filter: {
|
|
976
|
-
must: [
|
|
977
|
-
{
|
|
978
|
-
key: "url",
|
|
979
|
-
match: {
|
|
980
|
-
text: `${filePrefix}`
|
|
981
|
-
}
|
|
982
|
-
}
|
|
983
|
-
],
|
|
984
|
-
must_not: [
|
|
985
|
-
{
|
|
986
|
-
key: "is_metadata",
|
|
987
|
-
match: {
|
|
988
|
-
value: true
|
|
989
|
-
}
|
|
990
|
-
}
|
|
991
|
-
]
|
|
992
|
-
}
|
|
993
|
-
});
|
|
994
|
-
const obsoletePointIds = response.points
|
|
995
|
-
.filter((point) => {
|
|
996
|
-
const url = point.payload?.url;
|
|
997
|
-
// Double check it's not a metadata record
|
|
998
|
-
if (point.payload?.is_metadata === true) {
|
|
999
|
-
return false;
|
|
1000
|
-
}
|
|
1001
|
-
// Remove file:// prefix to match with processedFiles
|
|
1002
|
-
const filePath = url && url.startsWith('file://') ? url.substring(7) : '';
|
|
1003
|
-
return filePath && !processedFiles.has(filePath);
|
|
1004
|
-
})
|
|
1005
|
-
.map((point) => point.id);
|
|
1006
|
-
if (obsoletePointIds.length > 0) {
|
|
1007
|
-
await client.delete(collectionName, {
|
|
1008
|
-
points: obsoletePointIds,
|
|
1009
|
-
});
|
|
1010
|
-
logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for directory ${dirPrefix}`);
|
|
1011
|
-
}
|
|
1012
|
-
else {
|
|
1013
|
-
logger.info(`No obsolete chunks to delete from Qdrant for directory ${dirPrefix}`);
|
|
1014
|
-
}
|
|
1015
|
-
}
|
|
1016
|
-
catch (error) {
|
|
1017
|
-
logger.error(`Error removing obsolete chunks from Qdrant:`, error);
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
1100
|
isValidUuid(str) {
|
|
1021
1101
|
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
1022
1102
|
return uuidRegex.test(str);
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ACMCdkStack = void 0;
|
|
4
|
+
const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
5
|
+
class ACMCdkStack extends aws_cdk_lib_1.Stack {
|
|
6
|
+
constructor(scope, id, props) {
|
|
7
|
+
super(scope, id, props);
|
|
8
|
+
if (!process.env.DOMAIN_NAME) {
|
|
9
|
+
throw new Error("DOMAIN_NAME environment variable is not defined");
|
|
10
|
+
}
|
|
11
|
+
const hostedZone = props.hostedZone;
|
|
12
|
+
const edgeSiteDomainName = "apps." + process.env.DOMAIN_NAME;
|
|
13
|
+
this.acmCertificate = new aws_cdk_lib_1.aws_certificatemanager.Certificate(this, 'SiteCertificate', {
|
|
14
|
+
domainName: edgeSiteDomainName,
|
|
15
|
+
validation: aws_cdk_lib_1.aws_certificatemanager.CertificateValidation.fromDns(hostedZone),
|
|
16
|
+
});
|
|
17
|
+
new aws_cdk_lib_1.CfnOutput(this, 'ACMCertificate', { value: this.acmCertificate.certificateArn });
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
exports.ACMCdkStack = ACMCdkStack;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Route53CdkStack = void 0;
|
|
4
|
+
const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
5
|
+
class Route53CdkStack extends aws_cdk_lib_1.Stack {
|
|
6
|
+
constructor(scope, id, props) {
|
|
7
|
+
super(scope, id, props);
|
|
8
|
+
if (!process.env.PARENT_DOMAIN_NAME) {
|
|
9
|
+
throw new Error("PARENT_DOMAIN_NAME environment variable is not defined");
|
|
10
|
+
}
|
|
11
|
+
if (!process.env.DOMAIN_NAME) {
|
|
12
|
+
throw new Error("DOMAIN_NAME environment variable is not defined");
|
|
13
|
+
}
|
|
14
|
+
this.hostedZone = new aws_cdk_lib_1.aws_route53.PublicHostedZone(this, "HostedZone", {
|
|
15
|
+
zoneName: process.env.DOMAIN_NAME,
|
|
16
|
+
comment: "Managed by CDK"
|
|
17
|
+
});
|
|
18
|
+
const parent = aws_cdk_lib_1.aws_route53.HostedZone.fromLookup(this, "ParentHostedZone", {
|
|
19
|
+
domainName: process.env.PARENT_DOMAIN_NAME + "."
|
|
20
|
+
});
|
|
21
|
+
if (!this.hostedZone.hostedZoneNameServers) {
|
|
22
|
+
throw new Error("NS record is undefined.");
|
|
23
|
+
}
|
|
24
|
+
new aws_cdk_lib_1.aws_route53.ZoneDelegationRecord(this, "NS", {
|
|
25
|
+
zone: parent,
|
|
26
|
+
recordName: process.env.DOMAIN_NAME,
|
|
27
|
+
nameServers: this.hostedZone.hostedZoneNameServers,
|
|
28
|
+
comment: "Managed by CDK"
|
|
29
|
+
});
|
|
30
|
+
new aws_cdk_lib_1.CfnOutput(this, 'ZoneId', { value: this.hostedZone.hostedZoneId });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
exports.Route53CdkStack = Route53CdkStack;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.WAFCdkStack = void 0;
|
|
4
|
+
const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
5
|
+
class WAFCdkStack extends aws_cdk_lib_1.Stack {
|
|
6
|
+
constructor(scope, id, props) {
|
|
7
|
+
super(scope, id, props);
|
|
8
|
+
const webAcl = new aws_cdk_lib_1.aws_wafv2.CfnWebACL(this, 'RateLimitWebAcl', {
|
|
9
|
+
description: 'WebACL to rate limit the calls',
|
|
10
|
+
defaultAction: { allow: {} },
|
|
11
|
+
scope: 'REGIONAL',
|
|
12
|
+
visibilityConfig: {
|
|
13
|
+
metricName: 'RateLimitWebAclMetric',
|
|
14
|
+
cloudWatchMetricsEnabled: true,
|
|
15
|
+
sampledRequestsEnabled: true,
|
|
16
|
+
},
|
|
17
|
+
rules: [
|
|
18
|
+
{
|
|
19
|
+
// rate limit to 100 based on IP within 5 minutes
|
|
20
|
+
name: 'AWSRateLimitRuleSet',
|
|
21
|
+
priority: 1,
|
|
22
|
+
action: { block: {} },
|
|
23
|
+
visibilityConfig: {
|
|
24
|
+
metricName: "AWSRateLimitIP",
|
|
25
|
+
cloudWatchMetricsEnabled: true,
|
|
26
|
+
sampledRequestsEnabled: false
|
|
27
|
+
},
|
|
28
|
+
statement: {
|
|
29
|
+
rateBasedStatement: {
|
|
30
|
+
aggregateKeyType: "IP",
|
|
31
|
+
limit: 100,
|
|
32
|
+
evaluationWindowSec: 300
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
// common rule set
|
|
38
|
+
name: 'AWSManagedRulesCommonRuleSet',
|
|
39
|
+
priority: 2,
|
|
40
|
+
overrideAction: { none: {} },
|
|
41
|
+
visibilityConfig: {
|
|
42
|
+
metricName: 'AWSManagedRulesCommonRuleSetMetric',
|
|
43
|
+
cloudWatchMetricsEnabled: true,
|
|
44
|
+
sampledRequestsEnabled: true
|
|
45
|
+
},
|
|
46
|
+
statement: {
|
|
47
|
+
managedRuleGroupStatement: {
|
|
48
|
+
vendorName: 'AWS',
|
|
49
|
+
name: 'AWSManagedRulesCommonRuleSet'
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
]
|
|
54
|
+
});
|
|
55
|
+
const alb = aws_cdk_lib_1.aws_elasticloadbalancingv2.ApplicationLoadBalancer.fromLookup(this, 'ALBListener', {
|
|
56
|
+
loadBalancerTags: {
|
|
57
|
+
'elbv2.k8s.aws/cluster': process.env.CLUSTER_OWNER + '-' + process.env.PROJECT
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
new aws_cdk_lib_1.aws_wafv2.CfnWebACLAssociation(this, "ALBWebACLAssociation", {
|
|
61
|
+
webAclArn: webAcl.attrArn,
|
|
62
|
+
resourceArn: alb.loadBalancerArn
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
exports.WAFCdkStack = WAFCdkStack;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// import * as cdk from 'aws-cdk-lib';
|
|
3
|
+
// import { Template } from 'aws-cdk-lib/assertions';
|
|
4
|
+
// import * as S3CloudfrontCdk from '../lib/s3-cloudfront-cdk-stack';
|
|
5
|
+
// example test. To run these tests, uncomment this file along with the
|
|
6
|
+
// example resource in lib/s3-cloudfront-cdk-stack.ts
|
|
7
|
+
test('SQS Queue Created', () => {
|
|
8
|
+
// const app = new cdk.App();
|
|
9
|
+
// // WHEN
|
|
10
|
+
// const stack = new S3CloudfrontCdk.S3CloudfrontCdkStack(app, 'MyTestStack');
|
|
11
|
+
// // THEN
|
|
12
|
+
// const template = Template.fromStack(stack);
|
|
13
|
+
// template.hasResourceProperties('AWS::SQS::Queue', {
|
|
14
|
+
// VisibilityTimeout: 300
|
|
15
|
+
// });
|
|
16
|
+
});
|