@htilabs/platform-common-models 0.83.0 → 0.84.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,6 +3,10 @@ var text = (v) => typeof v === "string" ? v.trim() : "";
3
3
  function comparableName(value) {
4
4
  return text(value).normalize("NFKD").toLowerCase().replace(/&/g, " and ").replace(/[^a-z0-9]+/g, " ").trim().replace(/^the /, "");
5
5
  }
6
+ function comparableId(value) {
7
+ const s = value.trim();
8
+ return /^[0-9a-fA-F]{24}$/.test(s) ? s.toLowerCase() : s;
9
+ }
6
10
  var sheetText = (v) => {
7
11
  const s = typeof v === "string" ? v : v == null ? "" : String(v);
8
12
  return /^[=+\-@\t\r]/.test(s) ? `'${s}` : s;
@@ -30,6 +34,12 @@ var PROGRAM_STATUS_VALUES = Object.freeze([
30
34
  PROGRAM_STATUS.ACTIVE,
31
35
  PROGRAM_STATUS.RETIRED
32
36
  ]);
37
+ function isLiveOrganizationProgram(row) {
38
+ return row.status !== PROGRAM_STATUS.RETIRED && row.reviewStatus !== PROGRAM_REVIEW_STATUS.REJECTED;
39
+ }
40
+ function programNameKey(name) {
41
+ return name.trim().toLowerCase();
42
+ }
33
43
 
34
44
  // src/orgDirectory/orgRecord.ts
35
45
  var ORG_REVIEW_STATUS = PROGRAM_REVIEW_STATUS;
@@ -603,14 +613,14 @@ var blank = (v) => v == null || typeof v === "string" && v.trim() === "";
603
613
  var idText = (v) => directoryIdString(v) ?? String(v);
604
614
  var HEX_242 = /^[0-9a-fA-F]{24}$/;
605
615
  var message = (err) => err instanceof Error ? err.message : String(err);
606
- var idOf = (driver, v) => typeof v === "string" ? v.trim().toLowerCase() : v instanceof driver.ObjectId ? v.toHexString() : null;
616
+ var idOf = (driver, v) => typeof v === "string" ? comparableId(v) : v instanceof driver.ObjectId ? v.toHexString() : null;
607
617
  function idForms(driver, id) {
608
618
  const s = idText(id);
609
619
  return HEX_242.test(s) ? [new driver.ObjectId(s.toLowerCase()), s.toLowerCase()] : [s];
610
620
  }
611
621
  function inFormOf(driver, from, keptId) {
612
622
  const s = idText(keptId);
613
- return from instanceof driver.ObjectId && HEX_242.test(s) ? new driver.ObjectId(s.toLowerCase()) : s.toLowerCase();
623
+ return from instanceof driver.ObjectId && HEX_242.test(s) ? new driver.ObjectId(s.toLowerCase()) : comparableId(s);
614
624
  }
615
625
  var THREAD_SAMPLE_ID_PREFIX = "64e4ff1c2a7dcd9cb5f4a";
616
626
  var APP_LABEL = { platform: "the platform", thread: "Thread", pave: "PAVE" };
@@ -647,16 +657,33 @@ var ORG_REFERENCES = Object.freeze([
647
657
  ]);
648
658
  var APP_DATABASE_KINDS = ["thread", "pave"];
649
659
  var APP_MARKER = { thread: "caserecords", pave: "protectedpeople" };
660
+ var PAVE_ONLY_COLLECTIONS = ["programhousingenrollments", "disclosure_records", "reportingdestinationconfigs"];
650
661
  async function appDatabaseProblem(app, name, db) {
651
- const found = await db.listCollections({ name: APP_MARKER[app] }, { nameOnly: true }).toArray();
652
- return found.length ? null : `'${name}' has no ${APP_MARKER[app]} collection, so it is not a ${APP_LABEL[app]} database (check the name)`;
662
+ const has = async (collection) => (await db.listCollections({ name: collection }, { nameOnly: true }).toArray()).length > 0;
663
+ if (!await has(APP_MARKER[app])) {
664
+ return `'${name}' has no ${APP_MARKER[app]} collection, so it is not a ${APP_LABEL[app]} database (check the name)`;
665
+ }
666
+ let paves = null;
667
+ for (const c of PAVE_ONLY_COLLECTIONS) {
668
+ if (await has(c)) {
669
+ paves = c;
670
+ break;
671
+ }
672
+ }
673
+ if (app === "thread" && paves) {
674
+ return `'${name}' has a ${paves} collection, which only PAVE's database has, so it is not a Thread database (check the name)`;
675
+ }
676
+ if (app === "pave" && !paves) {
677
+ return `'${name}' has none of the collections only PAVE's database has (${PAVE_ONLY_COLLECTIONS.join(", ")}), so it is not a PAVE database (check the name)`;
678
+ }
679
+ return null;
653
680
  }
654
681
  var refPath = (r) => r.kind === "rows" || r.kind === "answers" ? `${r.path}.${String(r.field)}` : r.path;
655
682
  function getPath(doc, path) {
656
683
  return path.split(".").reduce((v, k) => v && typeof v === "object" ? v[k] : void 0, doc);
657
684
  }
658
685
  function rewriteIds(driver, list, copy, kept) {
659
- const keptHex = idText(kept).toLowerCase();
686
+ const keptHex = comparableId(idText(kept));
660
687
  let hasKept = list.some((v) => idOf(driver, v) === keptHex);
661
688
  const out = [];
662
689
  for (const v of list) {
@@ -693,7 +720,7 @@ function rewriteAnswers(driver, answers, field, copy, kept) {
693
720
  }
694
721
  async function findReferences(driver, dbs, orgId, keptId) {
695
722
  const out = [];
696
- const copy = idText(orgId).toLowerCase();
723
+ const copy = comparableId(idText(orgId));
697
724
  for (const d of dbs) {
698
725
  for (const ref of ORG_REFERENCES.filter((r) => r.app === d.app)) {
699
726
  if (ref.kind === "count") continue;
@@ -710,7 +737,7 @@ async function findReferences(driver, dbs, orgId, keptId) {
710
737
  const before = getPath(doc, ref.path);
711
738
  if (!Array.isArray(before)) continue;
712
739
  const after = ref.kind === "list" ? rewriteIds(driver, before, copy, keptId) : ref.kind === "rows" ? rewriteRows(driver, before, String(ref.field), copy, keptId) : rewriteAnswers(driver, before, String(ref.field), copy, keptId);
713
- out.push({ ...base, docId: doc._id, from: copy, to: idText(keptId).toLowerCase(), before, after });
740
+ out.push({ ...base, docId: doc._id, from: copy, to: comparableId(idText(keptId)), before, after });
714
741
  }
715
742
  }
716
743
  }
@@ -743,7 +770,11 @@ async function holdsMerged(db, ref) {
743
770
  }
744
771
  var MERGE = "merge";
745
772
  var YES = "yes";
746
- async function planMerges(dbs, lines, driver) {
773
+ async function planMerges(dbs, lines, driver, opts) {
774
+ if (opts?.mergedBy !== "steward" && opts?.mergedBy !== "agency") {
775
+ throw new Error("planMerges: say who is merging (mergedBy: 'steward' or 'agency')");
776
+ }
777
+ const byAgency = opts.mergedBy === "agency";
747
778
  const platform = dbs[0];
748
779
  const orgs = platform.db.collection(ORGANIZATIONS_COLLECTION);
749
780
  const agencies = platform.db.collection(AGENCIES_COLLECTION);
@@ -786,7 +817,7 @@ async function planMerges(dbs, lines, driver) {
786
817
  const skip = (detail) => {
787
818
  out.push({ ...base, status: "skip", detail, flags });
788
819
  };
789
- if (line2.duplicateId.trim().toLowerCase() === line2.keptId.trim().toLowerCase()) {
820
+ if (comparableId(line2.duplicateId) === comparableId(line2.keptId)) {
790
821
  skip("the copy and the kept organization are the same row");
791
822
  continue;
792
823
  }
@@ -817,15 +848,45 @@ async function planMerges(dbs, lines, driver) {
817
848
  skip("the copy is an Agency's own organization (it names an Agency, or an Agency points at it): that is not a copy");
818
849
  continue;
819
850
  }
851
+ let programs;
820
852
  if (Array.isArray(dup.programs) && dup.programs.length) {
821
- skip("the copy has programs; merging programs is not supported");
822
- continue;
853
+ if (byAgency) {
854
+ skip("the copy has programs; merging programs is not supported");
855
+ continue;
856
+ }
857
+ const stored = async (id) => (await orgs.findOne({ _id: id }, { projection: { programs: 1 }, promoteValues: false }))?.programs;
858
+ const [keptRows, dupRows] = [await stored(kept._id), await stored(dup._id)];
859
+ if (!Array.isArray(dupRows) || dupRows.length !== dup.programs.length) {
860
+ skip("the copy's programs changed while the plan was read; plan again");
861
+ continue;
862
+ }
863
+ const keptHad = Array.isArray(keptRows);
864
+ const keptBefore = keptHad ? keptRows : [];
865
+ const idOfProgram = (r) => r && typeof r === "object" ? text4(r.programId) : "";
866
+ const keptIds = new Set(keptBefore.map(idOfProgram).filter(Boolean));
867
+ const twin = dupRows.map(idOfProgram).find((id) => id && keptIds.has(id));
868
+ if (twin) {
869
+ skip(`a program of the copy has the same id as a program of the kept organization (${twin}); two programs cannot share an id`);
870
+ continue;
871
+ }
872
+ programs = { keptHad, keptBefore, moved: dupRows };
873
+ const liveName = (r) => {
874
+ const row = r && typeof r === "object" ? r : {};
875
+ return typeof row.name === "string" && isLiveOrganizationProgram(row) ? programNameKey(row.name) : "";
876
+ };
877
+ const keptNames = new Set(keptBefore.map(liveName).filter(Boolean));
878
+ const same = programs.moved.filter((r) => liveName(r) && keptNames.has(liveName(r))).map((r) => text4(r.name));
879
+ if (same.length) {
880
+ flags.push(
881
+ `${same.length} program(s) of the copy have the name of a live program of the kept organization (${same.join(", ")}); both are kept, as two programs`
882
+ );
883
+ }
823
884
  }
824
885
  if (!isListedOrganization(kept)) {
825
886
  skip("the kept organization is inactive or declined: nothing is merged into a row that is not offered");
826
887
  continue;
827
888
  }
828
- if (await isUnitOf(kept, idText(dup._id).toLowerCase())) {
889
+ if (await isUnitOf(kept, comparableId(idText(dup._id)))) {
829
890
  skip("the kept organization is a unit of the copy (the copy is on its chain of parent organizations)");
830
891
  continue;
831
892
  }
@@ -841,10 +902,17 @@ async function planMerges(dbs, lines, driver) {
841
902
  const state = typeof viewer?.state === "string" ? viewer.state : "";
842
903
  const community = typeof kept.community === "string" ? kept.community : "";
843
904
  if (!sameList && !(state !== "" && state === community)) {
844
- skip(
845
- `the kept organization could not be seen by the agency whose list the copy is in: the copy is ${base.duplicateList} (that agency's state ${JSON.stringify(state)}), the kept row is ${base.keptList} (community ${JSON.stringify(community)}). An agency sees another list's organization only when its state is exactly that organization's community. The kept row's list or community is corrected first, by a deliberate edit`
846
- );
847
- continue;
905
+ if (byAgency) {
906
+ skip(
907
+ `the kept organization could not be seen by the agency whose list the copy is in: the copy is ${base.duplicateList} (that agency's state ${JSON.stringify(state)}), the kept row is ${base.keptList} (community ${JSON.stringify(community)}). An agency sees another list's organization only when its state is exactly that organization's community. The kept row's list or community is corrected first, by a deliberate edit`
908
+ );
909
+ continue;
910
+ }
911
+ if (viewer) {
912
+ flags.push(
913
+ `until Thread and PAVE show every agency the one organization list, ${text4(viewer.name) || "the agency that added the copy"} may not see the kept organization's name on its records that move` + (programs ? `, or pick the ${programs.moved.length} program(s) that move to it` : "")
914
+ );
915
+ }
848
916
  }
849
917
  const keyed = !blank(dup.key);
850
918
  if (keyed) {
@@ -860,6 +928,12 @@ async function planMerges(dbs, lines, driver) {
860
928
  const k = `${r.db} ${r.collection}.${refPath(r)}`;
861
929
  moves.set(k, (moves.get(k) ?? 0) + 1);
862
930
  }
931
+ if (programs && unchecked.length) {
932
+ const where = unchecked.map((k) => `${APP_LABEL[k]}'s database`).join(" and ");
933
+ flags.push(
934
+ `${where} could not be checked: a record there that names the copy with one of its programs still names the copy, while the programs are the kept organization's now; run the merge again with every database to move those records`
935
+ );
936
+ }
863
937
  const why = [
864
938
  ...unchecked.length ? [`references in ${unchecked.map((k) => `${APP_LABEL[k]}'s database`).join(" and ")} could not be checked`] : [],
865
939
  ...grants ? [`${grants} sharing grant(s) name the copy, and sharing rules are never moved`] : []
@@ -868,7 +942,7 @@ async function planMerges(dbs, lines, driver) {
868
942
  out.push({
869
943
  ...base,
870
944
  status: "merge",
871
- detail: `${moves.size ? `moves ${[...moves].map(([k, n]) => `${n} in ${k}`).join(", ")}` : "nothing points at the copy"}; ${then}`,
945
+ detail: `${moves.size ? `moves ${[...moves].map(([k, n]) => `${n} in ${k}`).join(", ")}` : "nothing points at the copy"}; ${programs ? `its ${programs.moved.length} program(s) move to the kept organization; ` : ""}${then}`,
872
946
  flags,
873
947
  write: {
874
948
  duplicateId: dup._id,
@@ -878,7 +952,8 @@ async function planMerges(dbs, lines, driver) {
878
952
  duplicate: dup,
879
953
  refs,
880
954
  outcome,
881
- addsLegacyEntry: !listed
955
+ addsLegacyEntry: !listed,
956
+ ...programs ? { programs } : {}
882
957
  }
883
958
  });
884
959
  }
@@ -888,6 +963,7 @@ function formatPlannedMerge(p) {
888
963
  return ` ${p.status.padEnd(6)} ${p.line.duplicateId} ${p.duplicateName || "(no organization)"}${p.duplicateList ? ` (${p.duplicateList})` : ""} -> ${p.line.keptId} ${p.keptName || "(no organization)"}${p.keptList ? ` (${p.keptList})` : ""}
889
964
  ${p.detail}${p.flags.map((f) => ` [${f}]`).join("")}`;
890
965
  }
966
+ var COPY_KEEPS_PROGRAMS = ", and its programs were added to the kept organization while the copy still lists them: undo this merge and run it again";
891
967
  async function executeMerges(dbs, merges, now, driver) {
892
968
  const orgs = dbs[0].db.collection(ORGANIZATIONS_COLLECTION);
893
969
  const dbNamed = new Map(dbs.map((d) => [d.name, d.db]));
@@ -895,12 +971,29 @@ async function executeMerges(dbs, merges, now, driver) {
895
971
  for (const merge of merges) {
896
972
  let moved = 0;
897
973
  try {
974
+ const programs = merge.programs;
898
975
  const kept = await orgs.updateOne(
899
- { _id: merge.keptId, status: { $in: [null, ORGANIZATION_STATUS.ACTIVE] }, reviewStatus: { $ne: "rejected" } },
900
- { $addToSet: { "legacy_ids.merged_from": idText(merge.duplicateId) } }
976
+ {
977
+ _id: merge.keptId,
978
+ status: { $in: [null, ORGANIZATION_STATUS.ACTIVE] },
979
+ reviewStatus: { $ne: "rejected" },
980
+ // The copy's programs join the kept row's only while that list is still exactly as the plan read it
981
+ // (`null` matches a row that stores no list).
982
+ ...programs ? { programs: programs.keptHad ? programs.keptBefore : null } : {}
983
+ },
984
+ {
985
+ $addToSet: { "legacy_ids.merged_from": idText(merge.duplicateId) },
986
+ // Added to the end: the kept row's own program rows are not rewritten, not even re-typed.
987
+ ...programs ? programs.keptHad ? { $push: { programs: { $each: programs.moved } } } : { $set: { programs: programs.moved } } : {}
988
+ }
901
989
  );
902
990
  if (kept.matchedCount !== 1) {
903
- out.push({ merge, status: "changed", moved, detail: "the kept organization is gone or no longer offered since the plan; nothing was written" });
991
+ out.push({
992
+ merge,
993
+ status: "changed",
994
+ moved,
995
+ detail: `the kept organization is gone or no longer offered${programs ? ", or its programs changed," : ""} since the plan; nothing was written`
996
+ });
904
997
  continue;
905
998
  }
906
999
  let stale = 0;
@@ -913,14 +1006,14 @@ async function executeMerges(dbs, merges, now, driver) {
913
1006
  if (merge.outcome === "delete" && left === 0) {
914
1007
  const res2 = await orgs.deleteOne(asPlanned);
915
1008
  out.push(
916
- res2.deletedCount === 1 ? { merge, status: "deleted", moved } : { merge, status: "changed", moved, detail: "the copy changed since the plan and was left as it is; its references were moved" }
1009
+ res2.deletedCount === 1 ? { merge, status: "deleted", moved } : { merge, status: "changed", moved, detail: `the copy changed since the plan and was left as it is; its references were moved${programs ? COPY_KEEPS_PROGRAMS : ""}` }
917
1010
  );
918
1011
  continue;
919
1012
  }
920
- const res = await orgs.updateOne(asPlanned, { $set: { status: ORGANIZATION_STATUS.INACTIVE, updatedTime: now } });
1013
+ const res = await orgs.updateOne(asPlanned, { $set: { status: ORGANIZATION_STATUS.INACTIVE, updatedTime: now, ...programs ? { programs: [] } : {} } });
921
1014
  const why = merge.outcome === "delete" ? `${left} reference(s) still point at the copy${stale ? ` (${stale} changed since the plan)` : ""}, so it was not deleted` : left ? `${left} reference(s) still point at the copy` : void 0;
922
1015
  out.push(
923
- res.matchedCount === 1 ? { merge, status: "retired", moved, ...why ? { detail: why } : {} } : { merge, status: "changed", moved, detail: "the copy changed since the plan and was left as it is; its references were moved" }
1016
+ res.matchedCount === 1 ? { merge, status: "retired", moved, ...why ? { detail: why } : {} } : { merge, status: "changed", moved, detail: `the copy changed since the plan and was left as it is; its references were moved${programs ? COPY_KEEPS_PROGRAMS : ""}` }
924
1017
  );
925
1018
  } catch (err) {
926
1019
  out.push({ merge, status: "failed", moved, detail: message(err) });
@@ -941,7 +1034,15 @@ async function rollbackMerges(dbFor, images, opts, driver) {
941
1034
  let partial = false;
942
1035
  const before = image.duplicate;
943
1036
  const now = await orgs.findOne({ _id: image.duplicateId });
944
- const retiredHere = !!now && sameDocument(driver, now, { ...before, status: ORGANIZATION_STATUS.INACTIVE, updatedTime: images.appliedTime });
1037
+ const programs = image.programs;
1038
+ const retiredHere = !!now && sameDocument(driver, now, {
1039
+ ...before,
1040
+ status: ORGANIZATION_STATUS.INACTIVE,
1041
+ updatedTime: images.appliedTime,
1042
+ // A copy the merge kept as inactive was left with no programs (they moved).
1043
+ ...programs ? { programs: [] } : {}
1044
+ });
1045
+ let copyBack = true;
945
1046
  if (!now) {
946
1047
  said.push("put the deleted copy back exactly as it was");
947
1048
  if (!opts.dryRun) await orgs.insertOne(before);
@@ -952,6 +1053,7 @@ async function rollbackMerges(dbFor, images, opts, driver) {
952
1053
  said.push("the copy is as it was");
953
1054
  } else {
954
1055
  partial = true;
1056
+ copyBack = false;
955
1057
  said.push("the copy has changed since the merge: left as it is");
956
1058
  }
957
1059
  let back = 0;
@@ -966,6 +1068,27 @@ async function rollbackMerges(dbFor, images, opts, driver) {
966
1068
  said.push(`${back} of ${image.refs.length} reference(s) back to the copy${moved ? ` (${moved} changed since the merge: left alone)` : ""}`);
967
1069
  }
968
1070
  if (moved) partial = true;
1071
+ let programsBack = false;
1072
+ if (programs) {
1073
+ const merged = [...programs.keptBefore, ...programs.moved];
1074
+ const holds = await orgs.countDocuments({ _id: image.keptId, programs: merged }) === 1;
1075
+ const asBefore = await orgs.countDocuments({ _id: image.keptId, programs: programs.keptHad ? programs.keptBefore : null }) === 1;
1076
+ if (holds) {
1077
+ programsBack = true;
1078
+ said.push(`take the copy's ${programs.moved.length} program(s) off the kept organization`);
1079
+ if (!opts.dryRun) {
1080
+ await orgs.updateOne(
1081
+ { _id: image.keptId, programs: merged },
1082
+ programs.keptHad ? { $set: { programs: programs.keptBefore } } : { $unset: { programs: "" } }
1083
+ );
1084
+ }
1085
+ } else if (!asBefore) {
1086
+ partial = true;
1087
+ said.push(
1088
+ "the kept organization's programs have changed since the merge: left as they are" + (copyBack ? " (the copy has its programs back too, so the same program ids are on both organizations until one list is corrected)" : "")
1089
+ );
1090
+ }
1091
+ }
969
1092
  const entry = idText(image.duplicateId);
970
1093
  const listed = image.addsLegacyEntry ? await orgs.countDocuments({ _id: image.keptId, "legacy_ids.merged_from": entry }) : 0;
971
1094
  if (listed) {
@@ -978,7 +1101,7 @@ async function rollbackMerges(dbFor, images, opts, driver) {
978
1101
  } else if (!image.addsLegacyEntry) {
979
1102
  said.push("the copy's id stays on the kept organization's legacy_ids (an earlier run put it there)");
980
1103
  }
981
- const anything = !now || retiredHere || back > 0 || listed > 0;
1104
+ const anything = !now || retiredHere || back > 0 || listed > 0 || programsBack;
982
1105
  out.push({
983
1106
  image,
984
1107
  status: !anything ? "skip" : partial ? "partial" : "restore",
@@ -995,6 +1118,7 @@ export {
995
1118
  THREAD_SAMPLE_ID_PREFIX,
996
1119
  YES,
997
1120
  appDatabaseProblem,
1121
+ comparableId,
998
1122
  comparableName,
999
1123
  executeMerges,
1000
1124
  formatPlannedMerge,