koishi-plugin-best-cave 1.3.4 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.js +197 -245
- package/lib/utils/HashStorage.d.ts +48 -56
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -293,7 +293,6 @@ var IdManager = class {
|
|
|
293
293
|
);
|
|
294
294
|
await this.saveStatus();
|
|
295
295
|
this.initialized = true;
|
|
296
|
-
logger2.success("ID Manager initialized");
|
|
297
296
|
} catch (error) {
|
|
298
297
|
this.initialized = false;
|
|
299
298
|
logger2.error(`ID Manager initialization failed: ${error.message}`);
|
|
@@ -593,253 +592,131 @@ var ImageHasher = class {
|
|
|
593
592
|
};
|
|
594
593
|
|
|
595
594
|
// src/utils/HashStorage.ts
|
|
596
|
-
var
|
|
595
|
+
var import_crypto = __toESM(require("crypto"));
|
|
597
596
|
var logger3 = new import_koishi3.Logger("HashStorage");
|
|
598
|
-
var readFileAsync = (0, import_util.promisify)(fs3.readFile);
|
|
599
597
|
var HashStorage = class _HashStorage {
|
|
600
598
|
/**
|
|
601
|
-
*
|
|
602
|
-
* @param caveDir 回声洞数据目录路径
|
|
599
|
+
* 创建哈希存储实例
|
|
600
|
+
* @param caveDir - 回声洞数据目录路径
|
|
603
601
|
*/
|
|
604
602
|
constructor(caveDir) {
|
|
605
603
|
this.caveDir = caveDir;
|
|
604
|
+
this.filePath = path3.join(caveDir, "hash.json");
|
|
605
|
+
this.resourceDir = path3.join(caveDir, "resources");
|
|
606
|
+
this.caveFilePath = path3.join(caveDir, "cave.json");
|
|
606
607
|
}
|
|
607
608
|
static {
|
|
608
609
|
__name(this, "HashStorage");
|
|
609
610
|
}
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
static BATCH_SIZE = 50;
|
|
616
|
-
// 存储回声洞ID到图片哈希值的映射
|
|
617
|
-
hashes = /* @__PURE__ */ new Map();
|
|
618
|
-
// 初始化状态标志
|
|
611
|
+
filePath;
|
|
612
|
+
resourceDir;
|
|
613
|
+
caveFilePath;
|
|
614
|
+
imageHashes = /* @__PURE__ */ new Map();
|
|
615
|
+
textHashes = /* @__PURE__ */ new Map();
|
|
619
616
|
initialized = false;
|
|
620
|
-
get filePath() {
|
|
621
|
-
return path3.join(this.caveDir, _HashStorage.HASH_FILE);
|
|
622
|
-
}
|
|
623
|
-
get resourceDir() {
|
|
624
|
-
return path3.join(this.caveDir, "resources");
|
|
625
|
-
}
|
|
626
|
-
get caveFilePath() {
|
|
627
|
-
return path3.join(this.caveDir, _HashStorage.CAVE_FILE);
|
|
628
|
-
}
|
|
629
617
|
/**
|
|
630
618
|
* 初始化哈希存储
|
|
631
|
-
* 读取现有哈希数据或重新构建哈希值
|
|
632
619
|
* @throws 初始化失败时抛出错误
|
|
633
620
|
*/
|
|
634
621
|
async initialize() {
|
|
635
622
|
if (this.initialized) return;
|
|
636
623
|
try {
|
|
637
624
|
const hashData = await FileHandler.readJsonData(this.filePath).then((data) => data[0]).catch(() => null);
|
|
638
|
-
if (!hashData?.
|
|
639
|
-
this.hashes.clear();
|
|
625
|
+
if (!hashData?.imageHashes) {
|
|
640
626
|
await this.buildInitialHashes();
|
|
641
627
|
} else {
|
|
642
|
-
this.
|
|
643
|
-
Object.entries(hashData.hashes).map(([k, v]) => [Number(k), v])
|
|
644
|
-
);
|
|
628
|
+
this.loadHashData(hashData);
|
|
645
629
|
await this.updateMissingHashes();
|
|
646
630
|
}
|
|
647
631
|
this.initialized = true;
|
|
648
632
|
} catch (error) {
|
|
649
|
-
logger3.error(`
|
|
650
|
-
this.initialized = false;
|
|
633
|
+
logger3.error(`Hash storage initialization failed: ${error.message}`);
|
|
651
634
|
throw error;
|
|
652
635
|
}
|
|
653
636
|
}
|
|
654
637
|
/**
|
|
655
|
-
*
|
|
656
|
-
* @
|
|
638
|
+
* 加载哈希数据
|
|
639
|
+
* @param data - 要加载的哈希数据
|
|
640
|
+
* @private
|
|
657
641
|
*/
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
lastUpdated: (/* @__PURE__ */ new Date()).toISOString(),
|
|
662
|
-
entries: Array.from(this.hashes.entries()).map(([caveId, hashes]) => ({
|
|
663
|
-
caveId,
|
|
664
|
-
hashes
|
|
665
|
-
}))
|
|
666
|
-
};
|
|
642
|
+
loadHashData(data) {
|
|
643
|
+
this.imageHashes = new Map(Object.entries(data.imageHashes).map(([k, v]) => [Number(k), v]));
|
|
644
|
+
this.textHashes = new Map(Object.entries(data.textHashes || {}).map(([k, v]) => [Number(k), v]));
|
|
667
645
|
}
|
|
668
646
|
/**
|
|
669
|
-
*
|
|
670
|
-
* @param caveId 回声洞ID
|
|
671
|
-
* @param
|
|
647
|
+
* 更新指定回声洞的哈希值
|
|
648
|
+
* @param caveId - 回声洞ID
|
|
649
|
+
* @param type - 哈希类型(图像或文本)
|
|
650
|
+
* @param content - 要计算哈希的内容
|
|
672
651
|
*/
|
|
673
|
-
async
|
|
652
|
+
async updateHash(caveId, type, content) {
|
|
674
653
|
if (!this.initialized) await this.initialize();
|
|
675
654
|
try {
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
this.hashes.set(caveId, hashes);
|
|
681
|
-
logger3.info(`Added ${hashes.length} hashes for cave ${caveId}`);
|
|
682
|
-
} else {
|
|
683
|
-
this.hashes.delete(caveId);
|
|
684
|
-
logger3.info(`Deleted hashes for cave ${caveId}`);
|
|
685
|
-
}
|
|
686
|
-
await this.saveHashes();
|
|
687
|
-
} catch (error) {
|
|
688
|
-
logger3.error(`Failed to update hash (cave ${caveId}): ${error.message}`);
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
/**
|
|
692
|
-
* 更新所有回声洞的哈希值
|
|
693
|
-
* @param isInitialBuild 是否为初始构建
|
|
694
|
-
*/
|
|
695
|
-
async updateAllCaves(isInitialBuild = false) {
|
|
696
|
-
if (!this.initialized && !isInitialBuild) {
|
|
697
|
-
await this.initialize();
|
|
698
|
-
return;
|
|
699
|
-
}
|
|
700
|
-
try {
|
|
701
|
-
logger3.info("Starting full hash update...");
|
|
702
|
-
const caveData = await this.loadCaveData();
|
|
703
|
-
const cavesWithImages = caveData.filter(
|
|
704
|
-
(cave) => cave.elements?.some((el) => el.type === "img" && el.file)
|
|
705
|
-
);
|
|
706
|
-
this.hashes.clear();
|
|
707
|
-
let processedCount = 0;
|
|
708
|
-
const totalImages = cavesWithImages.length;
|
|
709
|
-
const processCave = /* @__PURE__ */ __name(async (cave) => {
|
|
710
|
-
const imgElements = cave.elements?.filter((el) => el.type === "img" && el.file) || [];
|
|
711
|
-
if (imgElements.length === 0) return;
|
|
712
|
-
try {
|
|
713
|
-
const hashes = await Promise.all(
|
|
714
|
-
imgElements.map(async (imgElement) => {
|
|
715
|
-
const filePath = path3.join(this.resourceDir, imgElement.file);
|
|
716
|
-
if (!fs3.existsSync(filePath)) {
|
|
717
|
-
logger3.warn(`Image file not found: ${filePath}`);
|
|
718
|
-
return null;
|
|
719
|
-
}
|
|
720
|
-
const imgBuffer = await readFileAsync(filePath);
|
|
721
|
-
return await ImageHasher.calculateHash(imgBuffer);
|
|
722
|
-
})
|
|
723
|
-
);
|
|
724
|
-
const validHashes = hashes.filter((hash) => hash !== null);
|
|
725
|
-
if (validHashes.length > 0) {
|
|
726
|
-
this.hashes.set(cave.cave_id, validHashes);
|
|
727
|
-
processedCount++;
|
|
728
|
-
if (processedCount % 100 === 0) {
|
|
729
|
-
logger3.info(`Progress: ${processedCount}/${totalImages}`);
|
|
730
|
-
}
|
|
731
|
-
}
|
|
732
|
-
} catch (error) {
|
|
733
|
-
logger3.error(`Failed to process cave ${cave.cave_id}: ${error.message}`);
|
|
734
|
-
}
|
|
735
|
-
}, "processCave");
|
|
736
|
-
await this.processBatch(cavesWithImages, processCave);
|
|
655
|
+
const hash = type === "image" ? await ImageHasher.calculateHash(content) : _HashStorage.hashText(content);
|
|
656
|
+
const hashMap = type === "image" ? this.imageHashes : this.textHashes;
|
|
657
|
+
const existingHashes = hashMap.get(caveId) || [];
|
|
658
|
+
hashMap.set(caveId, [...existingHashes, hash]);
|
|
737
659
|
await this.saveHashes();
|
|
738
|
-
logger3.success(`Update completed. Processed ${processedCount}/${totalImages} images`);
|
|
739
660
|
} catch (error) {
|
|
740
|
-
logger3.error(`
|
|
741
|
-
throw error;
|
|
661
|
+
logger3.error(`Failed to update ${type} hash for cave ${caveId}: ${error.message}`);
|
|
742
662
|
}
|
|
743
663
|
}
|
|
744
664
|
/**
|
|
745
|
-
*
|
|
746
|
-
* @param
|
|
747
|
-
* @param
|
|
748
|
-
* @
|
|
665
|
+
* 查找重复项
|
|
666
|
+
* @param type - 查找类型(图像或文本)
|
|
667
|
+
* @param hashes - 要查找的哈希值数组
|
|
668
|
+
* @param threshold - 相似度阈值,默认为1
|
|
669
|
+
* @returns 匹配结果数组
|
|
749
670
|
*/
|
|
750
|
-
async findDuplicates(
|
|
671
|
+
async findDuplicates(type, hashes, threshold = 1) {
|
|
751
672
|
if (!this.initialized) await this.initialize();
|
|
752
|
-
const
|
|
753
|
-
|
|
754
|
-
)
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
if (similarity >= threshold && similarity > maxSimilarity) {
|
|
765
|
-
maxSimilarity = similarity;
|
|
766
|
-
matchedCaveId = caveId;
|
|
767
|
-
if (Math.abs(similarity - 1) < Number.EPSILON) break;
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
if (Math.abs(maxSimilarity - 1) < Number.EPSILON) break;
|
|
673
|
+
const hashMap = type === "image" ? this.imageHashes : this.textHashes;
|
|
674
|
+
const calculateSimilarity = type === "image" ? ImageHasher.calculateSimilarity : (a, b) => a === b ? 1 : 0;
|
|
675
|
+
return hashes.map((hash, index) => {
|
|
676
|
+
let maxSimilarity = 0;
|
|
677
|
+
let matchedCaveId = null;
|
|
678
|
+
for (const [caveId, existingHashes] of hashMap.entries()) {
|
|
679
|
+
for (const existingHash of existingHashes) {
|
|
680
|
+
const similarity = calculateSimilarity(hash, existingHash);
|
|
681
|
+
if (similarity >= threshold && similarity > maxSimilarity) {
|
|
682
|
+
maxSimilarity = similarity;
|
|
683
|
+
matchedCaveId = caveId;
|
|
684
|
+
if (similarity === 1) break;
|
|
771
685
|
}
|
|
772
|
-
return matchedCaveId ? {
|
|
773
|
-
index,
|
|
774
|
-
caveId: matchedCaveId,
|
|
775
|
-
similarity: maxSimilarity
|
|
776
|
-
} : null;
|
|
777
|
-
} catch (error) {
|
|
778
|
-
logger3.warn(`处理图片 ${index} 失败: ${error.message}`);
|
|
779
|
-
return null;
|
|
780
686
|
}
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
* 加载回声洞数据
|
|
786
|
-
* @returns 回声洞数据数组
|
|
787
|
-
* @private
|
|
788
|
-
*/
|
|
789
|
-
async loadCaveData() {
|
|
790
|
-
const data = await FileHandler.readJsonData(this.caveFilePath);
|
|
791
|
-
return Array.isArray(data) ? data.flat() : [];
|
|
687
|
+
if (maxSimilarity === 1) break;
|
|
688
|
+
}
|
|
689
|
+
return matchedCaveId ? { index, caveId: matchedCaveId, similarity: maxSimilarity } : null;
|
|
690
|
+
});
|
|
792
691
|
}
|
|
793
692
|
/**
|
|
794
|
-
*
|
|
795
|
-
* @
|
|
693
|
+
* 清除指定回声洞的所有哈希值
|
|
694
|
+
* @param caveId - 回声洞ID
|
|
796
695
|
*/
|
|
797
|
-
async
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
await FileHandler.writeJsonData(this.filePath, [data]);
|
|
696
|
+
async clearHashes(caveId) {
|
|
697
|
+
if (!this.initialized) await this.initialize();
|
|
698
|
+
this.imageHashes.delete(caveId);
|
|
699
|
+
this.textHashes.delete(caveId);
|
|
700
|
+
await this.saveHashes();
|
|
803
701
|
}
|
|
804
702
|
/**
|
|
805
|
-
*
|
|
703
|
+
* 构建初始哈希值
|
|
806
704
|
* @private
|
|
807
705
|
*/
|
|
808
706
|
async buildInitialHashes() {
|
|
809
707
|
const caveData = await this.loadCaveData();
|
|
810
|
-
let
|
|
811
|
-
const
|
|
812
|
-
logger3.info(`Building hash data for ${totalImages} images...`);
|
|
708
|
+
let processedCount = 0;
|
|
709
|
+
const total = caveData.length;
|
|
813
710
|
for (const cave of caveData) {
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
const filePath = path3.join(this.resourceDir, imgElement.file);
|
|
820
|
-
if (!fs3.existsSync(filePath)) {
|
|
821
|
-
logger3.warn(`Image not found: ${filePath}`);
|
|
822
|
-
return null;
|
|
823
|
-
}
|
|
824
|
-
const imgBuffer = await fs3.promises.readFile(filePath);
|
|
825
|
-
const hash = await ImageHasher.calculateHash(imgBuffer);
|
|
826
|
-
processedImageCount++;
|
|
827
|
-
if (processedImageCount % 100 === 0) {
|
|
828
|
-
logger3.info(`Progress: ${processedImageCount}/${totalImages} images`);
|
|
829
|
-
}
|
|
830
|
-
return hash;
|
|
831
|
-
})
|
|
832
|
-
);
|
|
833
|
-
const validHashes = hashes.filter((hash) => hash !== null);
|
|
834
|
-
if (validHashes.length > 0) {
|
|
835
|
-
this.hashes.set(cave.cave_id, validHashes);
|
|
836
|
-
}
|
|
837
|
-
} catch (error) {
|
|
838
|
-
logger3.error(`Failed to process cave ${cave.cave_id}: ${error.message}`);
|
|
711
|
+
await this.processCaveHashes(cave);
|
|
712
|
+
await this.processCaveTextHashes(cave);
|
|
713
|
+
processedCount++;
|
|
714
|
+
if (processedCount % 100 === 0 || processedCount === total) {
|
|
715
|
+
logger3.info(`Processing caves: ${processedCount}/${total} (${Math.floor(processedCount / total * 100)}%)`);
|
|
839
716
|
}
|
|
840
717
|
}
|
|
841
718
|
await this.saveHashes();
|
|
842
|
-
logger3.
|
|
719
|
+
logger3.info(`Processed ${processedCount} caves with ${this.imageHashes.size} images and ${this.textHashes.size} texts`);
|
|
843
720
|
}
|
|
844
721
|
/**
|
|
845
722
|
* 更新缺失的哈希值
|
|
@@ -847,53 +724,97 @@ var HashStorage = class _HashStorage {
|
|
|
847
724
|
*/
|
|
848
725
|
async updateMissingHashes() {
|
|
849
726
|
const caveData = await this.loadCaveData();
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
return null;
|
|
861
|
-
}
|
|
862
|
-
const imgBuffer = await fs3.promises.readFile(filePath);
|
|
863
|
-
return ImageHasher.calculateHash(imgBuffer);
|
|
864
|
-
})
|
|
865
|
-
);
|
|
866
|
-
const validHashes = hashes.filter((hash) => hash !== null);
|
|
867
|
-
if (validHashes.length > 0) {
|
|
868
|
-
this.hashes.set(cave.cave_id, validHashes);
|
|
869
|
-
updatedCount++;
|
|
727
|
+
const missingImageCaves = caveData.filter((cave) => !this.imageHashes.has(cave.cave_id));
|
|
728
|
+
const missingTextCaves = caveData.filter((cave) => !this.textHashes.has(cave.cave_id));
|
|
729
|
+
const total = missingImageCaves.length + missingTextCaves.length;
|
|
730
|
+
if (total > 0) {
|
|
731
|
+
let processedCount = 0;
|
|
732
|
+
for (const cave of missingImageCaves) {
|
|
733
|
+
await this.processCaveHashes(cave);
|
|
734
|
+
processedCount++;
|
|
735
|
+
if (processedCount % 100 === 0 || processedCount === total) {
|
|
736
|
+
logger3.info(`Updating missing hashes: ${processedCount}/${total} (${Math.floor(processedCount / total * 100)}%)`);
|
|
870
737
|
}
|
|
871
|
-
} catch (error) {
|
|
872
|
-
logger3.error(`Failed to process cave ${cave.cave_id}: ${error.message}`);
|
|
873
738
|
}
|
|
739
|
+
for (const cave of missingTextCaves) {
|
|
740
|
+
await this.processCaveTextHashes(cave);
|
|
741
|
+
processedCount++;
|
|
742
|
+
if (processedCount % 100 === 0 || processedCount === total) {
|
|
743
|
+
logger3.info(`Updating missing hashes: ${processedCount}/${total} (${Math.floor(processedCount / total * 100)}%)`);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
await this.saveHashes();
|
|
747
|
+
logger3.info(`Updated ${missingImageCaves.length} missing images and ${missingTextCaves.length} missing texts`);
|
|
874
748
|
}
|
|
875
749
|
}
|
|
876
750
|
/**
|
|
877
|
-
*
|
|
878
|
-
* @param
|
|
879
|
-
* @param processor 处理函数
|
|
880
|
-
* @param batchSize 批处理大小
|
|
751
|
+
* 处理单个回声洞的哈希值
|
|
752
|
+
* @param cave - 回声洞数据
|
|
881
753
|
* @private
|
|
882
754
|
*/
|
|
883
|
-
async
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
755
|
+
async processCaveHashes(cave) {
|
|
756
|
+
const imgElements = cave.elements?.filter((el) => el.type === "img" && el.file) || [];
|
|
757
|
+
if (imgElements.length === 0) return;
|
|
758
|
+
try {
|
|
759
|
+
const hashes = await Promise.all(imgElements.map(async (el) => {
|
|
760
|
+
const filePath = path3.join(this.resourceDir, el.file);
|
|
761
|
+
return fs3.existsSync(filePath) ? ImageHasher.calculateHash(await fs3.promises.readFile(filePath)) : null;
|
|
762
|
+
}));
|
|
763
|
+
const validHashes = hashes.filter(Boolean);
|
|
764
|
+
if (validHashes.length) {
|
|
765
|
+
this.imageHashes.set(cave.cave_id, validHashes);
|
|
766
|
+
await this.saveHashes();
|
|
767
|
+
}
|
|
768
|
+
} catch (error) {
|
|
769
|
+
logger3.error(`Failed to process cave ${cave.cave_id}: ${error.message}`);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
async processCaveTextHashes(cave) {
|
|
773
|
+
const textElements = cave.elements?.filter((el) => el.type === "text" && el.content) || [];
|
|
774
|
+
if (textElements.length === 0) return;
|
|
775
|
+
try {
|
|
776
|
+
const hashes = textElements.map((el) => _HashStorage.hashText(el.content));
|
|
777
|
+
if (hashes.length) {
|
|
778
|
+
this.textHashes.set(cave.cave_id, hashes);
|
|
779
|
+
}
|
|
780
|
+
} catch (error) {
|
|
781
|
+
logger3.error(`Failed to process text hashes for cave ${cave.cave_id}: ${error.message}`);
|
|
895
782
|
}
|
|
896
783
|
}
|
|
784
|
+
/**
|
|
785
|
+
* 保存哈希数据到文件
|
|
786
|
+
* @private
|
|
787
|
+
*/
|
|
788
|
+
async saveHashes() {
|
|
789
|
+
try {
|
|
790
|
+
const data = {
|
|
791
|
+
imageHashes: Object.fromEntries(this.imageHashes),
|
|
792
|
+
textHashes: Object.fromEntries(this.textHashes),
|
|
793
|
+
lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
|
|
794
|
+
};
|
|
795
|
+
await FileHandler.writeJsonData(this.filePath, [data]);
|
|
796
|
+
} catch (error) {
|
|
797
|
+
logger3.error(`Failed to save hash data: ${error.message}`);
|
|
798
|
+
throw error;
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* 加载回声洞数据
|
|
803
|
+
* @returns 回声洞数据数组
|
|
804
|
+
* @private
|
|
805
|
+
*/
|
|
806
|
+
async loadCaveData() {
|
|
807
|
+
const data = await FileHandler.readJsonData(this.caveFilePath);
|
|
808
|
+
return Array.isArray(data) ? data.flat() : [];
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* 计算文本的哈希值
|
|
812
|
+
* @param text - 要计算哈希的文本
|
|
813
|
+
* @returns MD5哈希值
|
|
814
|
+
*/
|
|
815
|
+
static hashText(text) {
|
|
816
|
+
return import_crypto.default.createHash("md5").update(text).digest("hex");
|
|
817
|
+
}
|
|
897
818
|
};
|
|
898
819
|
|
|
899
820
|
// src/index.ts
|
|
@@ -1025,7 +946,7 @@ async function apply(ctx, config) {
|
|
|
1025
946
|
if (targetCave.elements) {
|
|
1026
947
|
const hashStorage2 = new HashStorage(caveDir);
|
|
1027
948
|
await hashStorage2.initialize();
|
|
1028
|
-
await hashStorage2.
|
|
949
|
+
await hashStorage2.clearHashes(caveId);
|
|
1029
950
|
for (const element of targetCave.elements) {
|
|
1030
951
|
if ((element.type === "img" || element.type === "video") && element.file) {
|
|
1031
952
|
const fullPath = path4.join(resourceDir, element.file);
|
|
@@ -1054,7 +975,7 @@ async function apply(ctx, config) {
|
|
|
1054
975
|
const inputContent = content.length > 0 ? content.join("\n") : await (async () => {
|
|
1055
976
|
await sendMessage(session, "commands.cave.add.noContent", [], true);
|
|
1056
977
|
const reply = await session.prompt({ timeout: 6e4 });
|
|
1057
|
-
if (!reply)
|
|
978
|
+
if (!reply) session.text("commands.cave.add.operationTimeout");
|
|
1058
979
|
return reply;
|
|
1059
980
|
})();
|
|
1060
981
|
const caveId = await idManager.getNextId();
|
|
@@ -1063,6 +984,20 @@ async function apply(ctx, config) {
|
|
|
1063
984
|
}
|
|
1064
985
|
const bypassAudit = config2.whitelist.includes(session.userId) || config2.whitelist.includes(session.guildId) || config2.whitelist.includes(session.channelId);
|
|
1065
986
|
const { imageUrls, imageElements, videoUrls, videoElements, textParts } = await extractMediaContent(inputContent, config2, session);
|
|
987
|
+
const pureText = textParts.filter((tp) => tp.type === "text").map((tp) => tp.content.trim()).join("\n").trim();
|
|
988
|
+
if (config2.enableMD5 && pureText) {
|
|
989
|
+
const textHash = HashStorage.hashText(pureText);
|
|
990
|
+
const textDuplicates = await hashStorage.findDuplicates("text", [textHash]);
|
|
991
|
+
if (textDuplicates[0]) {
|
|
992
|
+
const data2 = await FileHandler.readJsonData(caveFilePath);
|
|
993
|
+
const duplicateCave = data2.find((item) => item.cave_id === textDuplicates[0].caveId);
|
|
994
|
+
if (duplicateCave) {
|
|
995
|
+
const message = session.text("commands.cave.error.exactDuplicateFound");
|
|
996
|
+
await session.send(message + await buildMessage(duplicateCave, resourceDir, session));
|
|
997
|
+
throw new Error("duplicate_found");
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1066
1001
|
if (videoUrls.length > 0 && !config2.allowVideo) {
|
|
1067
1002
|
return sendMessage(session, "commands.cave.add.videoDisabled", [], true);
|
|
1068
1003
|
}
|
|
@@ -1098,7 +1033,7 @@ async function apply(ctx, config) {
|
|
|
1098
1033
|
// 保持原始文本和图片的相对位置
|
|
1099
1034
|
index: el.index
|
|
1100
1035
|
}))
|
|
1101
|
-
].sort((a
|
|
1036
|
+
].sort((a) => a.index - a.index),
|
|
1102
1037
|
contributor_number: session.userId,
|
|
1103
1038
|
contributor_name: session.username
|
|
1104
1039
|
};
|
|
@@ -1109,17 +1044,12 @@ async function apply(ctx, config) {
|
|
|
1109
1044
|
index: Number.MAX_SAFE_INTEGER
|
|
1110
1045
|
});
|
|
1111
1046
|
}
|
|
1112
|
-
const
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
(cave) => cave.elements?.some((element) => element.type === "img" && element.file)
|
|
1119
|
-
);
|
|
1120
|
-
if (hasImages) {
|
|
1121
|
-
await hashStorage2.updateAllCaves(true);
|
|
1122
|
-
}
|
|
1047
|
+
const existingData = await FileHandler.readJsonData(caveFilePath);
|
|
1048
|
+
const hasImages = existingData.some(
|
|
1049
|
+
(cave) => cave.elements?.some((element) => element.type === "img" && element.file)
|
|
1050
|
+
);
|
|
1051
|
+
if (hasImages) {
|
|
1052
|
+
await hashStorage.initialize();
|
|
1123
1053
|
}
|
|
1124
1054
|
if (config2.enableAudit && !bypassAudit) {
|
|
1125
1055
|
const pendingData = await FileHandler.readJsonData(pendingFilePath);
|
|
@@ -1137,9 +1067,14 @@ async function apply(ctx, config) {
|
|
|
1137
1067
|
});
|
|
1138
1068
|
await Promise.all([
|
|
1139
1069
|
FileHandler.writeJsonData(caveFilePath, data),
|
|
1140
|
-
|
|
1070
|
+
pureText && config2.enableMD5 ? hashStorage.updateHash(caveId, "text", pureText) : Promise.resolve(),
|
|
1071
|
+
savedImages?.length ? Promise.all(savedImages.map((buffer) => hashStorage.updateHash(caveId, "image", buffer))) : Promise.resolve()
|
|
1141
1072
|
]);
|
|
1142
1073
|
await idManager.addStat(session.userId, caveId);
|
|
1074
|
+
if (config2.enableMD5 && pureText) {
|
|
1075
|
+
const textHash = HashStorage.hashText(pureText);
|
|
1076
|
+
await hashStorage.updateHash(caveId, "text", textHash);
|
|
1077
|
+
}
|
|
1143
1078
|
return sendMessage(session, "commands.cave.add.addSuccess", [caveId], false);
|
|
1144
1079
|
} catch (error) {
|
|
1145
1080
|
logger4.error(`Failed to process add command: ${error.message}`);
|
|
@@ -1503,7 +1438,7 @@ async function saveMedia(urls, fileNames, resourceDir, caveId, mediaType, config
|
|
|
1503
1438
|
if (config.enableDuplicate) {
|
|
1504
1439
|
const hashStorage2 = new HashStorage(path4.join(ctx.baseDir, "data", "cave"));
|
|
1505
1440
|
await hashStorage2.initialize();
|
|
1506
|
-
const result = await hashStorage2.findDuplicates([buffer], config.duplicateThreshold);
|
|
1441
|
+
const result = await hashStorage2.findDuplicates("image", [buffer.toString("base64")], config.duplicateThreshold);
|
|
1507
1442
|
if (result.length > 0 && result[0] !== null) {
|
|
1508
1443
|
const duplicate = result[0];
|
|
1509
1444
|
const similarity = duplicate.similarity;
|
|
@@ -1528,6 +1463,23 @@ async function saveMedia(urls, fileNames, resourceDir, caveId, mediaType, config
|
|
|
1528
1463
|
return finalFileName;
|
|
1529
1464
|
} else {
|
|
1530
1465
|
const baseName = path4.basename(fileName || "video", ext).replace(/[^\u4e00-\u9fa5a-zA-Z0-9]/g, "");
|
|
1466
|
+
if (config.enableMD5) {
|
|
1467
|
+
const files = await fs4.promises.readdir(resourceDir);
|
|
1468
|
+
const duplicateFile = files.find((file) => file.startsWith(baseName + "_"));
|
|
1469
|
+
if (duplicateFile) {
|
|
1470
|
+
const duplicateCaveId = parseInt(duplicateFile.split("_")[1]);
|
|
1471
|
+
if (!isNaN(duplicateCaveId)) {
|
|
1472
|
+
const caveFilePath = path4.join(ctx.baseDir, "data", "cave", "cave.json");
|
|
1473
|
+
const data = await FileHandler.readJsonData(caveFilePath);
|
|
1474
|
+
const originalCave = data.find((item) => item.cave_id === duplicateCaveId);
|
|
1475
|
+
if (originalCave) {
|
|
1476
|
+
const message = session.text("commands.cave.error.exactDuplicateFound");
|
|
1477
|
+
await session.send(message + await buildMessage(originalCave, resourceDir, session));
|
|
1478
|
+
throw new Error("duplicate_found");
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1531
1483
|
const finalFileName = `${caveId}_${baseName}${ext}`;
|
|
1532
1484
|
const filePath = path4.join(resourceDir, finalFileName);
|
|
1533
1485
|
await FileHandler.saveMediaFile(filePath, buffer);
|
|
@@ -1,95 +1,87 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
|
|
4
|
-
interface HashStatus {
|
|
5
|
-
/** 最后更新时间戳 */
|
|
6
|
-
lastUpdated: string;
|
|
7
|
-
/** 所有回声洞的哈希值条目 */
|
|
8
|
-
entries: Array<{
|
|
9
|
-
caveId: number;
|
|
10
|
-
hashes: string[];
|
|
11
|
-
}>;
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* 图片哈希值存储管理类
|
|
15
|
-
* 负责管理和维护回声洞图片的哈希值
|
|
2
|
+
* 哈希存储管理类
|
|
3
|
+
* @class HashStorage
|
|
16
4
|
*/
|
|
17
5
|
export declare class HashStorage {
|
|
18
6
|
private readonly caveDir;
|
|
19
|
-
private
|
|
20
|
-
private
|
|
21
|
-
private
|
|
22
|
-
private
|
|
7
|
+
private readonly filePath;
|
|
8
|
+
private readonly resourceDir;
|
|
9
|
+
private readonly caveFilePath;
|
|
10
|
+
private imageHashes;
|
|
11
|
+
private textHashes;
|
|
23
12
|
private initialized;
|
|
24
13
|
/**
|
|
25
|
-
*
|
|
26
|
-
* @param caveDir 回声洞数据目录路径
|
|
14
|
+
* 创建哈希存储实例
|
|
15
|
+
* @param caveDir - 回声洞数据目录路径
|
|
27
16
|
*/
|
|
28
17
|
constructor(caveDir: string);
|
|
29
|
-
private get filePath();
|
|
30
|
-
private get resourceDir();
|
|
31
|
-
private get caveFilePath();
|
|
32
18
|
/**
|
|
33
19
|
* 初始化哈希存储
|
|
34
|
-
* 读取现有哈希数据或重新构建哈希值
|
|
35
20
|
* @throws 初始化失败时抛出错误
|
|
36
21
|
*/
|
|
37
22
|
initialize(): Promise<void>;
|
|
38
23
|
/**
|
|
39
|
-
*
|
|
40
|
-
* @
|
|
41
|
-
|
|
42
|
-
getStatus(): Promise<HashStatus>;
|
|
43
|
-
/**
|
|
44
|
-
* 更新指定回声洞的图片哈希值
|
|
45
|
-
* @param caveId 回声洞ID
|
|
46
|
-
* @param imgBuffers 图片buffer数组
|
|
24
|
+
* 加载哈希数据
|
|
25
|
+
* @param data - 要加载的哈希数据
|
|
26
|
+
* @private
|
|
47
27
|
*/
|
|
48
|
-
|
|
28
|
+
private loadHashData;
|
|
49
29
|
/**
|
|
50
|
-
*
|
|
51
|
-
* @param
|
|
30
|
+
* 更新指定回声洞的哈希值
|
|
31
|
+
* @param caveId - 回声洞ID
|
|
32
|
+
* @param type - 哈希类型(图像或文本)
|
|
33
|
+
* @param content - 要计算哈希的内容
|
|
52
34
|
*/
|
|
53
|
-
|
|
35
|
+
updateHash(caveId: number, type: 'image' | 'text', content: Buffer | string): Promise<void>;
|
|
54
36
|
/**
|
|
55
|
-
*
|
|
56
|
-
* @param
|
|
57
|
-
* @param
|
|
58
|
-
* @
|
|
37
|
+
* 查找重复项
|
|
38
|
+
* @param type - 查找类型(图像或文本)
|
|
39
|
+
* @param hashes - 要查找的哈希值数组
|
|
40
|
+
* @param threshold - 相似度阈值,默认为1
|
|
41
|
+
* @returns 匹配结果数组
|
|
59
42
|
*/
|
|
60
|
-
findDuplicates(
|
|
43
|
+
findDuplicates(type: 'image' | 'text', hashes: string[], threshold?: number): Promise<Array<{
|
|
61
44
|
index: number;
|
|
62
45
|
caveId: number;
|
|
63
46
|
similarity: number;
|
|
64
47
|
} | null>>;
|
|
65
48
|
/**
|
|
66
|
-
*
|
|
67
|
-
* @
|
|
49
|
+
* 清除指定回声洞的所有哈希值
|
|
50
|
+
* @param caveId - 回声洞ID
|
|
51
|
+
*/
|
|
52
|
+
clearHashes(caveId: number): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* 构建初始哈希值
|
|
68
55
|
* @private
|
|
69
56
|
*/
|
|
70
|
-
private
|
|
57
|
+
private buildInitialHashes;
|
|
71
58
|
/**
|
|
72
|
-
*
|
|
59
|
+
* 更新缺失的哈希值
|
|
73
60
|
* @private
|
|
74
61
|
*/
|
|
75
|
-
private
|
|
62
|
+
private updateMissingHashes;
|
|
76
63
|
/**
|
|
77
|
-
*
|
|
64
|
+
* 处理单个回声洞的哈希值
|
|
65
|
+
* @param cave - 回声洞数据
|
|
78
66
|
* @private
|
|
79
67
|
*/
|
|
80
|
-
private
|
|
68
|
+
private processCaveHashes;
|
|
69
|
+
private processCaveTextHashes;
|
|
81
70
|
/**
|
|
82
|
-
*
|
|
71
|
+
* 保存哈希数据到文件
|
|
83
72
|
* @private
|
|
84
73
|
*/
|
|
85
|
-
private
|
|
74
|
+
private saveHashes;
|
|
86
75
|
/**
|
|
87
|
-
*
|
|
88
|
-
* @
|
|
89
|
-
* @param processor 处理函数
|
|
90
|
-
* @param batchSize 批处理大小
|
|
76
|
+
* 加载回声洞数据
|
|
77
|
+
* @returns 回声洞数据数组
|
|
91
78
|
* @private
|
|
92
79
|
*/
|
|
93
|
-
private
|
|
80
|
+
private loadCaveData;
|
|
81
|
+
/**
|
|
82
|
+
* 计算文本的哈希值
|
|
83
|
+
* @param text - 要计算哈希的文本
|
|
84
|
+
* @returns MD5哈希值
|
|
85
|
+
*/
|
|
86
|
+
static hashText(text: string): string;
|
|
94
87
|
}
|
|
95
|
-
export {};
|