@ganju/utils 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channelDebounce.d.ts +11 -0
- package/dist/channelDebounce.d.ts.map +1 -1
- package/dist/channelDebounce.js +12 -1
- package/dist/chunking.d.ts +17 -3
- package/dist/chunking.d.ts.map +1 -1
- package/dist/chunking.js +125 -19
- package/dist/constants.d.ts +24 -0
- package/dist/constants.d.ts.map +1 -1
- package/dist/constants.js +102 -16
- package/dist/index.d.ts +76 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -0
- package/dist/llmBaseUrl.d.ts +39 -0
- package/dist/llmBaseUrl.d.ts.map +1 -0
- package/dist/llmBaseUrl.js +65 -0
- package/dist/localizeZodIssue.d.ts.map +1 -1
- package/dist/localizeZodIssue.js +6 -0
- package/dist/resourceRerank.d.ts +20 -0
- package/dist/resourceRerank.d.ts.map +1 -0
- package/dist/resourceRerank.js +37 -0
- package/dist/schema.d.ts +41 -24
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +47 -8
- package/dist/syncableResource.d.ts +9 -0
- package/dist/syncableResource.d.ts.map +1 -0
- package/dist/syncableResource.js +31 -0
- package/dist/toolCatalog.d.ts +1 -1
- package/dist/toolCatalog.js +1 -1
- package/package.json +1 -1
- package/src/channelDebounce.ts +36 -0
- package/src/chunking.ts +166 -23
- package/src/constants.ts +114 -16
- package/src/index.ts +26 -1
- package/src/llmBaseUrl.ts +65 -0
- package/src/localizeZodIssue.ts +6 -0
- package/src/resourceRerank.ts +54 -0
- package/src/schema.ts +80 -11
- package/src/syncableResource.ts +48 -0
- package/src/toolCatalog.ts +1 -1
package/dist/schema.d.ts
CHANGED
|
@@ -740,18 +740,6 @@ export declare const Schema: {
|
|
|
740
740
|
organizationId: z.ZodUUID;
|
|
741
741
|
}, z.core.$strip>;
|
|
742
742
|
ARTIFACT_CREATE_RESOURCE_VIEW: z.ZodObject<{
|
|
743
|
-
content: z.ZodOptional<z.ZodString>;
|
|
744
|
-
type: z.ZodDefault<z.ZodEnum<{
|
|
745
|
-
static: "static";
|
|
746
|
-
template: "template";
|
|
747
|
-
}>>;
|
|
748
|
-
description: z.ZodOptional<z.ZodString>;
|
|
749
|
-
title: z.ZodString;
|
|
750
|
-
uri: z.ZodString;
|
|
751
|
-
crawlConfig: z.ZodOptional<z.ZodObject<{
|
|
752
|
-
maxPages: z.ZodDefault<z.ZodNumber>;
|
|
753
|
-
maxDepth: z.ZodDefault<z.ZodNumber>;
|
|
754
|
-
}, z.core.$strip>>;
|
|
755
743
|
mimeType: z.ZodEnum<{
|
|
756
744
|
"text/plain": "text/plain";
|
|
757
745
|
"text/csv": "text/csv";
|
|
@@ -792,6 +780,19 @@ export declare const Schema: {
|
|
|
792
780
|
"application/x-perl": "application/x-perl";
|
|
793
781
|
"application/octet-stream": "application/octet-stream";
|
|
794
782
|
}>;
|
|
783
|
+
fileName: z.ZodOptional<z.ZodString>;
|
|
784
|
+
content: z.ZodOptional<z.ZodString>;
|
|
785
|
+
type: z.ZodDefault<z.ZodEnum<{
|
|
786
|
+
static: "static";
|
|
787
|
+
template: "template";
|
|
788
|
+
}>>;
|
|
789
|
+
description: z.ZodOptional<z.ZodString>;
|
|
790
|
+
title: z.ZodString;
|
|
791
|
+
uri: z.ZodString;
|
|
792
|
+
crawlConfig: z.ZodOptional<z.ZodObject<{
|
|
793
|
+
maxPages: z.ZodDefault<z.ZodNumber>;
|
|
794
|
+
maxDepth: z.ZodDefault<z.ZodNumber>;
|
|
795
|
+
}, z.core.$strip>>;
|
|
795
796
|
size: z.ZodOptional<z.ZodNumber>;
|
|
796
797
|
sourceType: z.ZodDefault<z.ZodEnum<{
|
|
797
798
|
FILE: "FILE";
|
|
@@ -802,7 +803,6 @@ export declare const Schema: {
|
|
|
802
803
|
}>>;
|
|
803
804
|
encoding: z.ZodOptional<z.ZodString>;
|
|
804
805
|
fileKey: z.ZodOptional<z.ZodString>;
|
|
805
|
-
fileName: z.ZodOptional<z.ZodString>;
|
|
806
806
|
annotations: z.ZodOptional<z.ZodObject<{
|
|
807
807
|
audience: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
808
808
|
user: "user";
|
|
@@ -1001,14 +1001,6 @@ export declare const Schema: {
|
|
|
1001
1001
|
organizationId: z.ZodUUID;
|
|
1002
1002
|
}, z.core.$strip>;
|
|
1003
1003
|
ARTIFACT_UPDATE_RESOURCE_VIEW: z.ZodObject<{
|
|
1004
|
-
content: z.ZodOptional<z.ZodString>;
|
|
1005
|
-
type: z.ZodDefault<z.ZodEnum<{
|
|
1006
|
-
static: "static";
|
|
1007
|
-
template: "template";
|
|
1008
|
-
}>>;
|
|
1009
|
-
description: z.ZodOptional<z.ZodString>;
|
|
1010
|
-
title: z.ZodString;
|
|
1011
|
-
uri: z.ZodString;
|
|
1012
1004
|
mimeType: z.ZodEnum<{
|
|
1013
1005
|
"text/plain": "text/plain";
|
|
1014
1006
|
"text/csv": "text/csv";
|
|
@@ -1049,6 +1041,15 @@ export declare const Schema: {
|
|
|
1049
1041
|
"application/x-perl": "application/x-perl";
|
|
1050
1042
|
"application/octet-stream": "application/octet-stream";
|
|
1051
1043
|
}>;
|
|
1044
|
+
fileName: z.ZodOptional<z.ZodString>;
|
|
1045
|
+
content: z.ZodOptional<z.ZodString>;
|
|
1046
|
+
type: z.ZodDefault<z.ZodEnum<{
|
|
1047
|
+
static: "static";
|
|
1048
|
+
template: "template";
|
|
1049
|
+
}>>;
|
|
1050
|
+
description: z.ZodOptional<z.ZodString>;
|
|
1051
|
+
title: z.ZodString;
|
|
1052
|
+
uri: z.ZodString;
|
|
1052
1053
|
size: z.ZodOptional<z.ZodNumber>;
|
|
1053
1054
|
sourceType: z.ZodDefault<z.ZodEnum<{
|
|
1054
1055
|
FILE: "FILE";
|
|
@@ -1059,7 +1060,6 @@ export declare const Schema: {
|
|
|
1059
1060
|
}>>;
|
|
1060
1061
|
encoding: z.ZodOptional<z.ZodString>;
|
|
1061
1062
|
fileKey: z.ZodOptional<z.ZodString>;
|
|
1062
|
-
fileName: z.ZodOptional<z.ZodString>;
|
|
1063
1063
|
annotations: z.ZodOptional<z.ZodObject<{
|
|
1064
1064
|
audience: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
1065
1065
|
user: "user";
|
|
@@ -1215,8 +1215,8 @@ export declare const Schema: {
|
|
|
1215
1215
|
"openai-compatible": "openai-compatible";
|
|
1216
1216
|
}>;
|
|
1217
1217
|
apiKey: z.ZodString;
|
|
1218
|
-
model: z.ZodString;
|
|
1219
1218
|
baseUrl: z.ZodUnion<[z.ZodOptional<z.ZodURL>, z.ZodLiteral<"">]>;
|
|
1219
|
+
model: z.ZodString;
|
|
1220
1220
|
systemPrompt: z.ZodOptional<z.ZodString>;
|
|
1221
1221
|
}, z.core.$strip>;
|
|
1222
1222
|
ORGANIZATION_UPDATE_LLM: z.ZodObject<{
|
|
@@ -1246,8 +1246,8 @@ export declare const Schema: {
|
|
|
1246
1246
|
"openai-compatible": "openai-compatible";
|
|
1247
1247
|
}>>;
|
|
1248
1248
|
apiKey: z.ZodOptional<z.ZodString>;
|
|
1249
|
-
model: z.ZodOptional<z.ZodString>;
|
|
1250
1249
|
baseUrl: z.ZodNullable<z.ZodUnion<[z.ZodOptional<z.ZodURL>, z.ZodLiteral<"">]>>;
|
|
1250
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1251
1251
|
systemPrompt: z.ZodNullable<z.ZodOptional<z.ZodString>>;
|
|
1252
1252
|
}, z.core.$strip>;
|
|
1253
1253
|
ORGANIZATION_REMOVE_LLM: z.ZodObject<{
|
|
@@ -1366,6 +1366,23 @@ export declare const Schema: {
|
|
|
1366
1366
|
userId: z.ZodUUID;
|
|
1367
1367
|
organizationId: z.ZodUUID;
|
|
1368
1368
|
}, z.core.$strip>;
|
|
1369
|
+
ARTIFACT_UPDATE_RESOURCE_SYNC_INTERVAL: z.ZodObject<{
|
|
1370
|
+
resourceId: z.ZodUUID;
|
|
1371
|
+
syncInterval: z.ZodEnum<{
|
|
1372
|
+
off: "off";
|
|
1373
|
+
daily: "daily";
|
|
1374
|
+
weekly: "weekly";
|
|
1375
|
+
}>;
|
|
1376
|
+
projectId: z.ZodUUID;
|
|
1377
|
+
userId: z.ZodUUID;
|
|
1378
|
+
organizationId: z.ZodUUID;
|
|
1379
|
+
}, z.core.$strip>;
|
|
1380
|
+
ARTIFACT_SYNC_RESOURCE: z.ZodObject<{
|
|
1381
|
+
resourceId: z.ZodUUID;
|
|
1382
|
+
projectId: z.ZodUUID;
|
|
1383
|
+
userId: z.ZodUUID;
|
|
1384
|
+
organizationId: z.ZodUUID;
|
|
1385
|
+
}, z.core.$strip>;
|
|
1369
1386
|
HTTP_ENDPOINT_CONFIG: z.ZodPipe<z.ZodObject<{
|
|
1370
1387
|
name: z.ZodString;
|
|
1371
1388
|
title: z.ZodOptional<z.ZodString>;
|
package/dist/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAixBxB,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0FrB,CAAC;AAyDN,QAAA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;iBAmCpB,CAAC;AAWH,QAAA,MAAM,kBAAkB;;;;;;;;;;iBA+CtB,CAAC;AAiCH,QAAA,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAapB,CAAC;AAYH,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgCxB,CAAC;AA4LH,QAAA,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA4DpC,CAAC;AA4BL,QAAA,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAkDzB,CAAC;AAuHH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4GlB,CAAC;AAGF,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAG1E,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAIlE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAMtE,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAGtE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAMtE,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAIvE,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAC5C,OAAO,kCAAkC,CAC1C,CAAC;AAKF,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAKD,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAKD,MAAM,WAAW,wBAAwB;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CAC1E;AAMD,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjD,KAAK,EAAE,sBAAsB,EAAE,CAAC;IAChC,SAAS,CAAC,EAAE,0BAA0B,EAAE,CAAC;IACzC,OAAO,CAAC,EAAE,wBAAwB,EAAE,CAAC;CACtC"}
|
package/dist/schema.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Schema = void 0;
|
|
4
4
|
const zod_1 = require("zod");
|
|
5
5
|
const constants_1 = require("./constants");
|
|
6
|
+
const llmBaseUrl_1 = require("./llmBaseUrl");
|
|
6
7
|
const slug_1 = require("./slug");
|
|
7
8
|
const reservedToolName_1 = require("./reservedToolName");
|
|
8
9
|
const slugifyTitle_1 = require("./slugifyTitle");
|
|
@@ -491,29 +492,52 @@ const ORGANIZATION_LIST_LLM = zod_1.z.object({
|
|
|
491
492
|
userId: zod_1.z.uuid(),
|
|
492
493
|
organizationId: zod_1.z.uuid()
|
|
493
494
|
});
|
|
494
|
-
|
|
495
|
+
// The two rules live in `llmBaseUrl.ts`, because the update controller applies
|
|
496
|
+
// them again to the merged row — the only place both halves are visible. Here
|
|
497
|
+
// they are just the field validator and the cross-field check.
|
|
498
|
+
const LLM_BASE_URL = zod_1.z
|
|
499
|
+
.url()
|
|
500
|
+
.max(500)
|
|
501
|
+
.refine(llmBaseUrl_1.isAllowedLlmBaseUrl, constants_1.constants.LLM_BASE_URL_INVALID_MESSAGE);
|
|
502
|
+
const BASE_URL_REQUIRED = {
|
|
503
|
+
message: constants_1.constants.LLM_BASE_URL_REQUIRED_MESSAGE,
|
|
504
|
+
path: ['baseUrl']
|
|
505
|
+
};
|
|
506
|
+
const hasBaseUrlWhenRequired = (provider, baseUrl) => !(0, llmBaseUrl_1.requiresLlmBaseUrl)(provider) || !!baseUrl;
|
|
507
|
+
// The base-URL rule is a cross-field check, and zod cannot `.omit()` from a
|
|
508
|
+
// schema that already carries one. So the shape stays a plain object — the
|
|
509
|
+
// `_VIEW` variants below derive from it — and the refinement is applied last,
|
|
510
|
+
// to each schema in turn, which also keeps the two in step.
|
|
511
|
+
const ORGANIZATION_CREATE_LLM_SHAPE = zod_1.z.object({
|
|
495
512
|
name: zod_1.z.string().min(1).max(200),
|
|
496
513
|
provider: zod_1.z.enum(constants_1.constants.LLM_PROVIDERS),
|
|
497
514
|
model: zod_1.z.string().min(1).max(200),
|
|
498
|
-
baseUrl:
|
|
515
|
+
baseUrl: LLM_BASE_URL.optional().or(zod_1.z.literal('')),
|
|
499
516
|
apiKey: zod_1.z.string().min(1).max(500),
|
|
500
517
|
systemPrompt: zod_1.z.string().max(10000).optional(),
|
|
501
518
|
config: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).optional(),
|
|
502
519
|
userId: zod_1.z.uuid(),
|
|
503
520
|
organizationId: zod_1.z.uuid()
|
|
504
521
|
});
|
|
505
|
-
const
|
|
522
|
+
const ORGANIZATION_CREATE_LLM = ORGANIZATION_CREATE_LLM_SHAPE.refine(v => hasBaseUrlWhenRequired(v.provider, v.baseUrl), BASE_URL_REQUIRED);
|
|
523
|
+
// An update is a partial: an omitted field keeps its stored value, so the
|
|
524
|
+
// base-URL rule can only be checked here when the caller sends the provider and
|
|
525
|
+
// the base URL together. The controller re-checks it against the merged row,
|
|
526
|
+
// which is the only place both halves are known.
|
|
527
|
+
const ORGANIZATION_UPDATE_LLM_SHAPE = zod_1.z.object({
|
|
506
528
|
llmId: zod_1.z.uuid(),
|
|
507
529
|
name: zod_1.z.string().min(1).max(200).optional(),
|
|
508
530
|
provider: zod_1.z.enum(constants_1.constants.LLM_PROVIDERS).optional(),
|
|
509
531
|
model: zod_1.z.string().min(1).max(200).optional(),
|
|
510
|
-
baseUrl:
|
|
532
|
+
baseUrl: LLM_BASE_URL.optional().or(zod_1.z.literal('')).nullable(),
|
|
511
533
|
apiKey: zod_1.z.string().min(1).max(500).optional(),
|
|
512
534
|
systemPrompt: zod_1.z.string().max(10000).optional().nullable(),
|
|
513
535
|
config: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).optional().nullable(),
|
|
514
536
|
userId: zod_1.z.uuid(),
|
|
515
537
|
organizationId: zod_1.z.uuid()
|
|
516
538
|
});
|
|
539
|
+
const refineUpdateBaseUrl = (schema) => schema.refine(v => v.baseUrl === undefined || hasBaseUrlWhenRequired(v.provider, v.baseUrl), BASE_URL_REQUIRED);
|
|
540
|
+
const ORGANIZATION_UPDATE_LLM = refineUpdateBaseUrl(ORGANIZATION_UPDATE_LLM_SHAPE);
|
|
517
541
|
const ORGANIZATION_REMOVE_LLM = zod_1.z.object({
|
|
518
542
|
llmId: zod_1.z.uuid(),
|
|
519
543
|
userId: zod_1.z.uuid(),
|
|
@@ -561,15 +585,15 @@ const ACCESS_TOKEN_CREATE_VIEW = ACCESS_TOKEN_CREATE.omit({
|
|
|
561
585
|
organizationId: true,
|
|
562
586
|
projectId: true
|
|
563
587
|
});
|
|
564
|
-
const ORGANIZATION_CREATE_LLM_VIEW =
|
|
588
|
+
const ORGANIZATION_CREATE_LLM_VIEW = ORGANIZATION_CREATE_LLM_SHAPE.omit({
|
|
565
589
|
userId: true,
|
|
566
590
|
organizationId: true
|
|
567
|
-
});
|
|
568
|
-
const ORGANIZATION_UPDATE_LLM_VIEW =
|
|
591
|
+
}).refine(v => hasBaseUrlWhenRequired(v.provider, v.baseUrl), BASE_URL_REQUIRED);
|
|
592
|
+
const ORGANIZATION_UPDATE_LLM_VIEW = refineUpdateBaseUrl(ORGANIZATION_UPDATE_LLM_SHAPE.omit({
|
|
569
593
|
llmId: true,
|
|
570
594
|
userId: true,
|
|
571
595
|
organizationId: true
|
|
572
|
-
});
|
|
596
|
+
}));
|
|
573
597
|
// `channel.config` is an open bag, so this stays loose — it only pins down the
|
|
574
598
|
// keys we actually read, and passes everything else through untouched.
|
|
575
599
|
// `debounceMs` is how long a burst from one participant is buffered before the
|
|
@@ -1239,6 +1263,19 @@ const ARTIFACT_UPDATE_RESOURCE_SHOW_SOURCE = zod_1.z.object({
|
|
|
1239
1263
|
userId: zod_1.z.uuid(),
|
|
1240
1264
|
organizationId: zod_1.z.uuid()
|
|
1241
1265
|
});
|
|
1266
|
+
const ARTIFACT_UPDATE_RESOURCE_SYNC_INTERVAL = zod_1.z.object({
|
|
1267
|
+
resourceId: zod_1.z.uuid(),
|
|
1268
|
+
syncInterval: zod_1.z.enum(constants_1.constants.RESOURCE_SYNC_INTERVALS),
|
|
1269
|
+
projectId: zod_1.z.uuid(),
|
|
1270
|
+
userId: zod_1.z.uuid(),
|
|
1271
|
+
organizationId: zod_1.z.uuid()
|
|
1272
|
+
});
|
|
1273
|
+
const ARTIFACT_SYNC_RESOURCE = zod_1.z.object({
|
|
1274
|
+
resourceId: zod_1.z.uuid(),
|
|
1275
|
+
projectId: zod_1.z.uuid(),
|
|
1276
|
+
userId: zod_1.z.uuid(),
|
|
1277
|
+
organizationId: zod_1.z.uuid()
|
|
1278
|
+
});
|
|
1242
1279
|
const ARTIFACT_GET = zod_1.z.object({
|
|
1243
1280
|
projectId: zod_1.z.uuid(),
|
|
1244
1281
|
userId: zod_1.z.uuid(),
|
|
@@ -1401,6 +1438,8 @@ exports.Schema = {
|
|
|
1401
1438
|
CHANNEL_LIST_CONVERSATIONS,
|
|
1402
1439
|
CHANNEL_LIST_MESSAGES,
|
|
1403
1440
|
ARTIFACT_UPDATE_RESOURCE_SHOW_SOURCE,
|
|
1441
|
+
ARTIFACT_UPDATE_RESOURCE_SYNC_INTERVAL,
|
|
1442
|
+
ARTIFACT_SYNC_RESOURCE,
|
|
1404
1443
|
HTTP_ENDPOINT_CONFIG,
|
|
1405
1444
|
HTTP_ENDPOINT_CONFIG_WRITE,
|
|
1406
1445
|
MCP_PROXY_CONFIG,
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export interface SyncableResource {
|
|
2
|
+
sourceType?: string | null;
|
|
3
|
+
parentResourceId?: string | null;
|
|
4
|
+
metadata?: unknown;
|
|
5
|
+
}
|
|
6
|
+
export type ResourceSyncProvider = 'website' | 'google-drive' | 'one-drive';
|
|
7
|
+
export declare const resourceSyncProvider: (resource: SyncableResource) => ResourceSyncProvider | null;
|
|
8
|
+
export declare const isSyncableResource: (resource: SyncableResource) => boolean;
|
|
9
|
+
//# sourceMappingURL=syncableResource.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"syncableResource.d.ts","sourceRoot":"","sources":["../src/syncableResource.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,MAAM,oBAAoB,GAAG,SAAS,GAAG,cAAc,GAAG,WAAW,CAAC;AAE5E,eAAO,MAAM,oBAAoB,GAC/B,UAAU,gBAAgB,KACzB,oBAAoB,GAAG,IAqBzB,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAI,UAAU,gBAAgB,KAAG,OACvB,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isSyncableResource = exports.resourceSyncProvider = void 0;
|
|
4
|
+
const constants_1 = require("./constants");
|
|
5
|
+
const resourceSyncProvider = (resource) => {
|
|
6
|
+
if (resource.parentResourceId)
|
|
7
|
+
return null;
|
|
8
|
+
switch (resource.sourceType) {
|
|
9
|
+
case constants_1.constants.RESOURCE_SOURCE_TYPE_WEBSITE:
|
|
10
|
+
return 'website';
|
|
11
|
+
case constants_1.constants.RESOURCE_SOURCE_TYPE_GOOGLE_DRIVE_FOLDER:
|
|
12
|
+
return 'google-drive';
|
|
13
|
+
case constants_1.constants.RESOURCE_SOURCE_TYPE_ONE_DRIVE_FOLDER:
|
|
14
|
+
return 'one-drive';
|
|
15
|
+
case constants_1.constants.RESOURCE_SOURCE_TYPE_FILE: {
|
|
16
|
+
const meta = resource.metadata && typeof resource.metadata === 'object'
|
|
17
|
+
? resource.metadata
|
|
18
|
+
: null;
|
|
19
|
+
if (typeof meta?.driveFileId === 'string')
|
|
20
|
+
return 'google-drive';
|
|
21
|
+
if (typeof meta?.oneDriveItemId === 'string')
|
|
22
|
+
return 'one-drive';
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
default:
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
exports.resourceSyncProvider = resourceSyncProvider;
|
|
30
|
+
const isSyncableResource = (resource) => (0, exports.resourceSyncProvider)(resource) !== null;
|
|
31
|
+
exports.isSyncableResource = isSyncableResource;
|
package/dist/toolCatalog.d.ts
CHANGED
|
@@ -350,7 +350,7 @@ export declare const TOOL_CATALOG: readonly [{
|
|
|
350
350
|
}, {
|
|
351
351
|
readonly key: "search-resources";
|
|
352
352
|
readonly title: "Search Resources";
|
|
353
|
-
readonly description: "Find the resources most relevant to a question
|
|
353
|
+
readonly description: "Find the resources most relevant to a question, by meaning and by exact keywords.";
|
|
354
354
|
readonly requiredScopes: null;
|
|
355
355
|
}, {
|
|
356
356
|
readonly key: "send-resource";
|
package/dist/toolCatalog.js
CHANGED
|
@@ -431,7 +431,7 @@ exports.TOOL_CATALOG = [
|
|
|
431
431
|
{
|
|
432
432
|
key: 'search-resources',
|
|
433
433
|
title: 'Search Resources',
|
|
434
|
-
description: 'Find the resources most relevant to a question
|
|
434
|
+
description: 'Find the resources most relevant to a question, by meaning and by exact keywords.',
|
|
435
435
|
requiredScopes: null
|
|
436
436
|
},
|
|
437
437
|
{
|
package/package.json
CHANGED
package/src/channelDebounce.ts
CHANGED
|
@@ -12,8 +12,41 @@ export interface BufferedChannelMessage {
|
|
|
12
12
|
text: string;
|
|
13
13
|
externalMessageId: string | null;
|
|
14
14
|
receivedAt: number;
|
|
15
|
+
// Answer without waiting for the burst to settle: an explicit command, or a
|
|
16
|
+
// message on a channel with buffering turned off. It still goes through the
|
|
17
|
+
// buffer — that is what runs the turn under a caller patient enough for a
|
|
18
|
+
// long answer — it just flushes at once, taking any pending text with it.
|
|
19
|
+
immediate?: boolean;
|
|
20
|
+
// The prompt this message invoked, when it was a slash command.
|
|
21
|
+
command?: BufferedChannelCommand | null;
|
|
22
|
+
// Where this message must be answered, when that isn't the envelope's
|
|
23
|
+
// `delivery` — a Discord interaction waiting on its deferred response. Kept
|
|
24
|
+
// on the message because the envelope is replaced by every newer push, and a
|
|
25
|
+
// plain message typed a moment later must not orphan the interaction.
|
|
26
|
+
reply?: Record<string, unknown> | null;
|
|
15
27
|
}
|
|
16
28
|
|
|
29
|
+
// A resolved slash-command prompt, carried with the message that invoked it so
|
|
30
|
+
// the flush can run it exactly as the webhook would have.
|
|
31
|
+
export interface BufferedChannelCommand {
|
|
32
|
+
promptId: string;
|
|
33
|
+
artifactPromptId: string | null;
|
|
34
|
+
promptTitle: string;
|
|
35
|
+
args: Record<string, string>;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// The prompt a batch runs: the newest command in it, since that is the one the
|
|
39
|
+
// participant asked for last. Null for a batch of plain text.
|
|
40
|
+
export const batchCommand = (
|
|
41
|
+
messages: BufferedChannelMessage[]
|
|
42
|
+
): BufferedChannelCommand | null => {
|
|
43
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
44
|
+
const command = messages[i].command;
|
|
45
|
+
if (command) return command;
|
|
46
|
+
}
|
|
47
|
+
return null;
|
|
48
|
+
};
|
|
49
|
+
|
|
17
50
|
// Everything the flush needs to run the turn and deliver the reply, minus the
|
|
18
51
|
// message texts themselves. The newest push wins: a display name or thread id
|
|
19
52
|
// can change mid-burst, and the freshest is the one to answer against.
|
|
@@ -35,6 +68,9 @@ export interface ChannelBufferEnvelope {
|
|
|
35
68
|
export interface ChannelBufferFlush {
|
|
36
69
|
envelope: ChannelBufferEnvelope;
|
|
37
70
|
messages: BufferedChannelMessage[];
|
|
71
|
+
// Stable across every hand-off of the same batch, so the worker can tell a
|
|
72
|
+
// retry of a turn it already answered from a new one.
|
|
73
|
+
batchId: string;
|
|
38
74
|
}
|
|
39
75
|
|
|
40
76
|
// Read the channel's debounce window off its `config`, clamped to the supported
|
package/src/chunking.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { constants } from './constants';
|
|
2
|
+
import { decodeEntities } from './sanitize';
|
|
2
3
|
|
|
3
4
|
import type {
|
|
4
5
|
ExtractedDocument,
|
|
@@ -50,24 +51,140 @@ export const splitRecursive = (
|
|
|
50
51
|
return chunks;
|
|
51
52
|
};
|
|
52
53
|
|
|
53
|
-
|
|
54
|
+
// Each piece keeps its `raw` text — the slice of the input it covers, without
|
|
55
|
+
// the overlap borrowed from the piece before — so a caller can walk the input
|
|
56
|
+
// in order without reading any of it twice.
|
|
57
|
+
const chunkPieces = (
|
|
54
58
|
text: string,
|
|
55
59
|
options?: { targetChars?: number; overlapChars?: number }
|
|
56
|
-
): string
|
|
60
|
+
): Array<{ raw: string; content: string }> => {
|
|
57
61
|
const target = options?.targetChars ?? constants.CHUNK_TARGET_CHARS;
|
|
58
62
|
const overlap = options?.overlapChars ?? constants.CHUNK_OVERLAP_CHARS;
|
|
59
63
|
|
|
60
64
|
const trimmed = text.trim();
|
|
61
65
|
if (!trimmed) return [];
|
|
62
|
-
if (trimmed.length <= target) return [trimmed];
|
|
66
|
+
if (trimmed.length <= target) return [{ raw: trimmed, content: trimmed }];
|
|
63
67
|
|
|
64
68
|
const rawChunks = splitRecursive(trimmed, target);
|
|
65
|
-
const
|
|
69
|
+
const pieces: Array<{ raw: string; content: string }> = [];
|
|
66
70
|
for (let i = 0; i < rawChunks.length; i++) {
|
|
67
71
|
const prev = i > 0 ? rawChunks[i - 1].slice(-overlap) : '';
|
|
68
|
-
|
|
72
|
+
pieces.push({
|
|
73
|
+
raw: rawChunks[i],
|
|
74
|
+
content: (prev ? prev + '\n' : '') + rawChunks[i]
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
return pieces.filter(p => p.content.trim().length > 0);
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
export const chunkText = (
|
|
81
|
+
text: string,
|
|
82
|
+
options?: { targetChars?: number; overlapChars?: number }
|
|
83
|
+
): string[] => chunkPieces(text, options).map(p => p.content);
|
|
84
|
+
|
|
85
|
+
export type HeadingSyntax = 'markdown' | 'html';
|
|
86
|
+
|
|
87
|
+
interface HeadingEvent {
|
|
88
|
+
offset: number;
|
|
89
|
+
level: number;
|
|
90
|
+
text: string;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const MARKDOWN_HEADING = /^ {0,3}(#{1,6})[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$/;
|
|
94
|
+
const MARKDOWN_FENCE = /^ {0,3}(```|~~~)/;
|
|
95
|
+
const HTML_HEADING = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1\s*>/gi;
|
|
96
|
+
const MAX_HEADING_CHARS = 120;
|
|
97
|
+
|
|
98
|
+
const MARKDOWN_FILE_NAME = /\.(md|markdown|mdx)$/i;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Which heading syntax a document is written in, or null when it has none we
|
|
102
|
+
* can trust. Plain text is left out on purpose: a line that happens to start
|
|
103
|
+
* with "# " there is as likely a comment or a list as a title.
|
|
104
|
+
*/
|
|
105
|
+
export const headingSyntaxFor = (input: {
|
|
106
|
+
mimeType?: string | null;
|
|
107
|
+
fileName?: string | null;
|
|
108
|
+
isWebPage?: boolean;
|
|
109
|
+
}): HeadingSyntax | null => {
|
|
110
|
+
// Crawled pages are flattened to text with their headings written back as
|
|
111
|
+
// Markdown lines, whatever the page's own content type was.
|
|
112
|
+
if (input.isWebPage) return 'markdown';
|
|
113
|
+
if (input.mimeType === constants.MIMETYPE_TEXT_MARKDOWN) return 'markdown';
|
|
114
|
+
if (input.mimeType === constants.MIMETYPE_TEXT_HTML) return 'html';
|
|
115
|
+
if (input.fileName && MARKDOWN_FILE_NAME.test(input.fileName)) {
|
|
116
|
+
return 'markdown';
|
|
117
|
+
}
|
|
118
|
+
return null;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
const cleanHeading = (text: string): string =>
|
|
122
|
+
text.replace(/\s+/g, ' ').trim().slice(0, MAX_HEADING_CHARS);
|
|
123
|
+
|
|
124
|
+
const scanMarkdownHeadings = (
|
|
125
|
+
text: string,
|
|
126
|
+
state: { inFence: boolean }
|
|
127
|
+
): HeadingEvent[] => {
|
|
128
|
+
const events: HeadingEvent[] = [];
|
|
129
|
+
let offset = 0;
|
|
130
|
+
for (const line of text.split('\n')) {
|
|
131
|
+
// A `#` inside a fenced block is a shell comment or a preprocessor line,
|
|
132
|
+
// not a title — and a fence can open in one chunk and close in the next.
|
|
133
|
+
if (MARKDOWN_FENCE.test(line)) {
|
|
134
|
+
state.inFence = !state.inFence;
|
|
135
|
+
} else if (!state.inFence) {
|
|
136
|
+
const match = MARKDOWN_HEADING.exec(line);
|
|
137
|
+
const heading = match ? cleanHeading(match[2]) : '';
|
|
138
|
+
if (match && heading) {
|
|
139
|
+
events.push({ offset, level: match[1].length, text: heading });
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
offset += line.length + 1;
|
|
69
143
|
}
|
|
70
|
-
return
|
|
144
|
+
return events;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
const scanHtmlHeadings = (text: string): HeadingEvent[] =>
|
|
148
|
+
[...text.matchAll(HTML_HEADING)]
|
|
149
|
+
.map(match => ({
|
|
150
|
+
offset: match.index ?? 0,
|
|
151
|
+
level: Number(match[1]),
|
|
152
|
+
text: cleanHeading(decodeEntities(match[2].replace(/<[^>]*>/g, ' ')))
|
|
153
|
+
}))
|
|
154
|
+
.filter(event => event.text);
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Follows the heading hierarchy across a document's chunks, fed in order.
|
|
158
|
+
* Returns, for each chunk, the path of headings it sits under — the section a
|
|
159
|
+
* reader would be pointed to. A chunk that opens with a heading belongs to that
|
|
160
|
+
* heading; one that starts mid-section belongs to the section it continues,
|
|
161
|
+
* even when a later heading begins inside it.
|
|
162
|
+
*/
|
|
163
|
+
const createHeadingTracker = (syntax: HeadingSyntax) => {
|
|
164
|
+
let stack: Array<{ level: number; text: string }> = [];
|
|
165
|
+
const fence = { inFence: false };
|
|
166
|
+
const apply = (event: HeadingEvent) => {
|
|
167
|
+
stack = stack.filter(h => h.level < event.level);
|
|
168
|
+
stack.push({ level: event.level, text: event.text });
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
return (raw: string): string[] => {
|
|
172
|
+
const events =
|
|
173
|
+
syntax === 'markdown'
|
|
174
|
+
? scanMarkdownHeadings(raw, fence)
|
|
175
|
+
: scanHtmlHeadings(raw);
|
|
176
|
+
const [first, ...rest] = events;
|
|
177
|
+
// Before a document's first heading there is no section to continue — only
|
|
178
|
+
// the metadata header, or the markup around an HTML body — so a chunk that
|
|
179
|
+
// holds the first heading belongs to it wherever in the chunk it falls.
|
|
180
|
+
const opensWithHeading =
|
|
181
|
+
first !== undefined &&
|
|
182
|
+
(stack.length === 0 || raw.slice(0, first.offset).trim() === '');
|
|
183
|
+
if (opensWithHeading) apply(first);
|
|
184
|
+
const path = stack.map(h => h.text);
|
|
185
|
+
for (const event of opensWithHeading ? rest : events) apply(event);
|
|
186
|
+
return path;
|
|
187
|
+
};
|
|
71
188
|
};
|
|
72
189
|
|
|
73
190
|
export const buildHeader = (input: {
|
|
@@ -87,9 +204,14 @@ export const buildHeader = (input: {
|
|
|
87
204
|
.filter(Boolean)
|
|
88
205
|
.join('\n');
|
|
89
206
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
207
|
+
// Loose on purpose: a chunk of inline text has no extracted document behind it,
|
|
208
|
+
// so only `headingPath` can be known about it.
|
|
209
|
+
export interface ChunkMetadata extends Partial<ExtractedDocumentMetadata> {
|
|
210
|
+
chunkIndexInPage?: number;
|
|
211
|
+
chunksInPage?: number;
|
|
212
|
+
// The headings this chunk sits under, outermost first. Present only for
|
|
213
|
+
// Markdown and HTML, and only when the chunk is under at least one heading.
|
|
214
|
+
headingPath?: string[];
|
|
93
215
|
}
|
|
94
216
|
|
|
95
217
|
export interface PreparedChunk {
|
|
@@ -97,28 +219,43 @@ export interface PreparedChunk {
|
|
|
97
219
|
metadata: ChunkMetadata | null;
|
|
98
220
|
}
|
|
99
221
|
|
|
222
|
+
const CONTROL_CHARS = /[\x00-\x08\x0B\x0C\x0E-\x1F]/g;
|
|
223
|
+
|
|
100
224
|
export const prepareChunks = (
|
|
101
225
|
header: string,
|
|
102
226
|
documents: ExtractedDocument[] | null,
|
|
103
227
|
fallbackContent: string | null,
|
|
104
|
-
options?: {
|
|
228
|
+
options?: {
|
|
229
|
+
targetChars?: number;
|
|
230
|
+
overlapChars?: number;
|
|
231
|
+
// The resource's own type and file name, which decide the heading syntax
|
|
232
|
+
// for inline content and for documents that don't carry their own.
|
|
233
|
+
mimeType?: string | null;
|
|
234
|
+
fileName?: string | null;
|
|
235
|
+
}
|
|
105
236
|
): PreparedChunk[] => {
|
|
106
237
|
if (documents && documents.length > 0) {
|
|
107
238
|
const out: PreparedChunk[] = [];
|
|
108
239
|
documents.forEach((doc, docIndex) => {
|
|
109
|
-
const body = (doc.pageContent || '')
|
|
110
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
111
|
-
.trim();
|
|
240
|
+
const body = (doc.pageContent || '').replace(CONTROL_CHARS, '').trim();
|
|
112
241
|
if (!body) return;
|
|
113
242
|
const text = docIndex === 0 && header ? `${header}\n\n${body}` : body;
|
|
114
|
-
const
|
|
243
|
+
const syntax = headingSyntaxFor({
|
|
244
|
+
mimeType: doc.metadata?.source?.mimeType ?? options?.mimeType,
|
|
245
|
+
fileName: doc.metadata?.source?.fileName ?? options?.fileName,
|
|
246
|
+
isWebPage: !!doc.metadata?.web
|
|
247
|
+
});
|
|
248
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
249
|
+
const pieces = chunkPieces(text, options);
|
|
115
250
|
pieces.forEach((piece, pieceIndex) => {
|
|
251
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
116
252
|
out.push({
|
|
117
|
-
content: piece,
|
|
253
|
+
content: piece.content,
|
|
118
254
|
metadata: {
|
|
119
255
|
...doc.metadata,
|
|
120
256
|
chunkIndexInPage: pieceIndex,
|
|
121
|
-
chunksInPage: pieces.length
|
|
257
|
+
chunksInPage: pieces.length,
|
|
258
|
+
...(headingPath.length > 0 ? { headingPath } : {})
|
|
122
259
|
}
|
|
123
260
|
});
|
|
124
261
|
});
|
|
@@ -126,12 +263,18 @@ export const prepareChunks = (
|
|
|
126
263
|
return out;
|
|
127
264
|
}
|
|
128
265
|
|
|
129
|
-
const body = (fallbackContent || '')
|
|
130
|
-
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, '')
|
|
131
|
-
.trim();
|
|
266
|
+
const body = (fallbackContent || '').replace(CONTROL_CHARS, '').trim();
|
|
132
267
|
const fullText = [header, body].filter(Boolean).join('\n\n');
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
})
|
|
268
|
+
const syntax = headingSyntaxFor({
|
|
269
|
+
mimeType: options?.mimeType,
|
|
270
|
+
fileName: options?.fileName
|
|
271
|
+
});
|
|
272
|
+
const headingPathOf = syntax ? createHeadingTracker(syntax) : null;
|
|
273
|
+
return chunkPieces(fullText, options).map(piece => {
|
|
274
|
+
const headingPath = headingPathOf?.(piece.raw) ?? [];
|
|
275
|
+
return {
|
|
276
|
+
content: piece.content,
|
|
277
|
+
metadata: headingPath.length > 0 ? { headingPath } : null
|
|
278
|
+
};
|
|
279
|
+
});
|
|
137
280
|
};
|