@bicharts/shape-core 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/*! @bicharts/shape-core — Apache-2.0. Bundled reference data: GeoNames (https://www.geonames.org/) CC BY 4.0; US Census/TIGER (public domain). Full text: NOTICE in this package. */
|
|
2
|
-
import{a as Y,b as G,c as x}from"./chunk-NWDPA7DG.mjs";import{a as U}from"./chunk-GY54KQLP.mjs";import{a as z,b as F,c as k}from"./chunk-WBEROMBG.mjs";import{a as H}from"./chunk-QEH7WTLB.mjs";import{a as j,b as q}from"./chunk-2LR73DNS.mjs";import X from"papaparse";var Z=12,$=100,J=.3,Q=.9,tt=120,et=1900,nt=2100,it=0,st=20,rt=new Set(["id","ids","uuid","guid","key","zip","zipcode","postal","postalcode","ssn","code","codes","sku","isbn","msisdn","imei"]);function at(p){if(!p)return!1;let e=p.replace(/([a-z0-9])([A-Z])/g,"$1 $2").replace(/[_\-./]+/g," ").toLowerCase().split(/\s+/).filter(o=>o.length>0);return e.length===0?!1:rt.has(e[e.length-1])}var ot=/\b(year|yr|fy|fiscal\s*year|calendar\s*year)\b/i,ut=/^(?:(?:19|20)\d{2}(?:[-/ ]?(?:q[1-4]|w(?:0?[1-9]|[1-4]\d|5[0-3])|(?:0?[1-9]|1[0-2])))?|q[1-4][-/ ](?:19|20)\d{2}|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*[-/ ]?(?:19|20)?\d{2})$/i;function W(p){if(!Number.isInteger(p)||p<190001||p>210012)return!1;let e=p%100;return e>=1&&e<=12}function P(p){if(!Number.isInteger(p)||p<19000101||p>21001231)return!1;let e=Math.floor(p/100)%100,o=p%100;return e>=1&&e<=12&&o>=1&&o<=31}function lt(p,e){if(!e||!/\b(?:19|20)\d{2}\b/.test(p))return!1;let o=p.replace(/\b(?:19|20)\d{2}\b/g,"").replace(/[-/.,]/g," ").trim();if(!o)return!1;let c=j(o);return c.length>0&&e[c]!==void 0}function ct(p){let{dataType:e,name:o,isMeasure:c,distinctCount:f}=p;if(e==="DateTime")return!0;if(c)return!1;if(e==="Integer"||e==="Decimal"){let n=p.minNum,b=p.maxNum;if(!(n!=null&&b!=null&&Number.isFinite(n)&&Number.isFinite(b)&&Number.isInteger(n)&&Number.isInteger(b)&&b>=n))return!1;let T=n>=1900&&b<=2100;if(T&&f>=2&&ot.test(o)||T&&f>=3&&f===b-n+1)return!0;let M=W(n)&&W(b),D=P(n)&&P(b);if(M||D){if(!p.sampleValues)return!0;let h=0,N=0;for(let l of p.sampleValues){if(l==null||l==="")continue;h++;let t=Number(String(l).trim());if(Number.isFinite(t)&&(M?W(t):P(t))&&N++,h>=60)break}if(h>=2&&N===h)return!0}return!1}if(p.sampleValues){let n=p.locale?q(p.locale):null,b=0,I=0;for(let T of p.sampleValues){if(T==null||T==="")continue;b++;let M=String(T).trim();if((ut.test(M)||lt(M,n))&&I++,b>=60)break}if(b>=2&&I/b>=.8)return!0}return!1}function mt(p){let e=(p||"").trim().toLowerCase();return e.startsWith("sum ")||e.startsWith("total ")?"sum":e.startsWith("average ")||e.startsWith("avg. ")?"avg":e.startsWith("count ")||e.startsWith("distinct count ")||e.startsWith("count (distinct) ")?"count":e.startsWith("min ")||e.startsWith("minimum ")?"min":e.startsWith("max ")||e.startsWith("maximum ")?"max":null}function ft(p){let{rows:e,measureColIdx:o,measureName:c,candidateDims:f,minNum:n,maxNum:b,discourageAgg:I}=p,T=100,M=.01,D="";for(let N of f){if(N.distinct<2||N.distinct>T)continue;let l=f.filter(i=>i.idx!==N.idx).map(i=>i.idx),t=new Map;for(let i of e){let u=i[o];if(typeof u!="number"||!Number.isFinite(u))continue;let d=l.length?l.map(y=>String(i[y])).join(D):"",g=t.get(d);g||(g={sum:0,cset:new Set},t.set(d,g)),g.sum+=u,g.cset.add(String(i[N.idx]))}let r=0,a=0,s=0;for(let i of t.values())i.cset.size<2||(s++,Math.abs(i.sum-100)<=100*M?r++:Math.abs(i.sum-1)<=1*M&&a++);if(s>=2&&(r/s>=.9||a/s>=.9))return{additivity:"part_of_whole",partOfWholeDim:N.name}}if(I)return{additivity:"intensive_rate"};let h=mt(c);return h==="avg"||h==="min"||h==="max"?{additivity:"intensive_rate"}:h==="sum"||h==="count"?{additivity:"additive"}:n!=null&&b!=null&&Number.isFinite(n)&&Number.isFinite(b)&&n>=0&&b<=1.0001&&b>0?{additivity:"intensive_rate"}:{additivity:"unknown"}}function B(p){let{dataType:e,isMeasure:o,name:c,distinct:f,nonblank:n,prec:b,maxval:I,minval:T}=p,M=I-T+1,D=M>0?f/M:1,h=n>0?f/n:0,N=e==="Decimal"||b>0;return at(c)?"Categorical":D>=Q&&(z(c)&&f<=tt||T>=et&&I<=nt||T>=it&&I<=st)?"Ordinal":o||f>Z&&(N||f>=$||h>=J)?"Continuous":"Categorical"}var K=class{_computedStatsForLevel=null;_cols=[];_rows=[];_origIndices=[];_rowHashes=new Set;STR(e){return e==null?"":e.toString()}getPrecision(e){if(!isFinite(e))return 0;let c=e.toString().split(".")[1];return c?c.length:0}getColumns(){return this._cols}hasTimeComponent(e){return!(e instanceof Date)||isNaN(e.getTime())?!1:e.getHours()!==0||e.getMinutes()!==0||e.getSeconds()!==0||e.getMilliseconds()!==0}getColumnsWithStats(e,o){if(this._computedStatsForLevel==e)return this._cols;let c=0,f=parseInt(e),n=[];for(let t of this._cols){let r=new Map,a=[],s=0,i=0,u=0,d=null,g=null,y=null,w=!1,R=!0,A=!0,v=null,S=!1;for(let C of this._rows){let m=C[c],_=this.STR(m);if(_!=""){if(s+=_.length,r.set(_,(r.get(_)||0)+1),i++,t.dataType=="Integer"||t.dataType=="Decimal"||t.dataType=="DateTime")if((d===null||m<d)&&(d=m),(g===null||m>g)&&(g=m),t.dataType=="Integer"||t.dataType=="Decimal"){if(y===null?y=m:y+=m,t.dataType=="Decimal"){let L=this.getPrecision(m);L>u&&(u=L)}a.push(m)}else w=w||this.hasTimeComponent(m);S&&(m<v?R=!1:m>v&&(A=!1)),v=m,S=!0}}if(i>=3&&(R&&!A?t.sortedDirection="asc":A&&!R?t.sortedDirection="desc":t.sortedDirection="none"),i>0&&r.size>0&&!t.isMeasure){let C=0,m=Number.MAX_SAFE_INTEGER;for(let _ of r.values())_>C&&(C=_),_<m&&(m=_);if(t.modalShare=C/i,t.minGroupCount=m===Number.MAX_SAFE_INTEGER?null:m,r.size===2){let _=[...r.keys()].map(V=>Y(V).trim().toLowerCase()).sort(),E=[["false","true"],["no","yes"],["n","y"],["0","1"],["f","t"]].some(V=>_[0]===V[0]&&_[1]===V[1]),O=/^(is|has)[_a-z0-9]|(?:default|churn|fraud|approv|convert|active|cancel|delinquen|flag|paid|win|pass|fail)/i.test(t.name);(E||O)&&(t.isBinaryFlag=!0)}}t.dataType=="DateTime"&&(t.dateWithTime=w),t.isTemporal=ct({dataType:t.dataType,name:t.name,isMeasure:t.isMeasure,distinctCount:r.size,minNum:typeof d=="number"?d:null,maxNum:typeof g=="number"?g:null,sampleValues:r.keys(),locale:o}),this.updateColumnStats10(f,t,i,r,u,g,d),this.updateColumnStats20(f,a,i,y,t,s,w,d,g,u,r,o),!t.isMeasure&&(t.dataType==="Integer"||t.dataType==="Decimal")&&B({dataType:t.dataType,isMeasure:!1,name:t.name,distinct:r.size,nonblank:i,prec:u,maxval:typeof g=="number"?g:0,minval:typeof d=="number"?d:0})==="Continuous"&&(t.isMeasure=!0),n[c]=t.isMeasure||r.size>2e3?null:new Set(r.keys()),c++}let b=[];this._cols.forEach((t,r)=>{t.isMeasure||b.push({idx:r,distinct:t.distinctCount??0,name:t.name})}),this._cols.forEach((t,r)=>{if(!t.isMeasure)return;let a=typeof t.lowValue=="number"?t.lowValue:null,s=typeof t.highValue=="number"?t.highValue:null,i=ft({rows:this._rows,measureColIdx:r,measureName:t.name,candidateDims:b,minNum:a,maxNum:s,discourageAgg:!!t.discourageAggregationAcrossGroups});t.additivity=i.additivity,i.partOfWholeDim&&(t.partOfWholeDim=i.partOfWholeDim)});let I=50,T=b.filter(t=>t.distinct>=2&&t.distinct<=I);this._cols.forEach((t,r)=>{if(!t.isMeasure)return;let a=[];for(let v of this._rows){let S=v[r];if(S==null||S==="")continue;let C=typeof S=="number"?S:parseFloat(S);isNaN(C)||a.push(C)}if(a.length===0)return;let s=[...a].sort((v,S)=>v-S),i=v=>s[Math.min(s.length-1,Math.max(0,Math.floor(v*(s.length-1))))],u=i(.5),d=i(.1),g=i(.9),y=Math.abs(u)>1e-9?Math.abs(u):Math.abs(s[s.length-1])>1e-9?Math.abs(s[s.length-1]):1;t.relativeDispersion=Math.round((g-d)/y*1e3)/1e3;let w=0;for(let v of a)w+=v;w/=a.length;let R=0;for(let v of a)R+=(v-w)*(v-w);if(R<=1e-12)return;let A=[];for(let v of T){let S=new Map;for(let m of this._rows){let _=m[r];if(_==null||_==="")continue;let L=typeof _=="number"?_:parseFloat(_);if(isNaN(L))continue;let E=this.STR(m[v.idx])+"",O=S.get(E);O?(O.s+=L,O.n++):S.set(E,{s:L,n:1})}let C=0;for(let m of S.values()){let _=m.s/m.n;C+=m.n*(_-w)*(_-w)}A.push({otherColumn:v.name,eta2:Math.round(C/R*1e4)/1e4})}A.length>0&&(t.groupDiscrimination=A)});let M=.97,D=5,h=this._cols.map((t,r)=>({c:t,i:r})).filter(t=>t.c.isMeasure);for(let t=0;t<h.length;t++)for(let r=t+1;r<h.length;r++){let a=h[t].i,s=h[r].i,i=0,u=0,d=0,g=0,y=0,w=0;for(let S of this._rows){let C=S[a],m=S[s];if(C==null||C===""||m===null||m===void 0||m==="")continue;let _=typeof C=="number"?C:parseFloat(C),L=typeof m=="number"?m:parseFloat(m);isNaN(_)||isNaN(L)||(i++,u+=_,d+=L,g+=_*_,y+=L*L,w+=_*L)}if(i<D)continue;let R=i*g-u*u,A=i*y-d*d;if(R<=1e-12||A<=1e-12)continue;let v=(i*w-u*d)/Math.sqrt(R*A);if(Math.abs(v)>=M){let S=h[t].c,C=h[r].c;S.collinearWithMeasures||(S.collinearWithMeasures=[]),C.collinearWithMeasures||(C.collinearWithMeasures=[]),S.collinearWithMeasures.push(C.name),C.collinearWithMeasures.push(S.name)}}for(let t=0;t<this._cols.length;t++){let r=n[t];if(!r||r.size===0)continue;let a=[];for(let s=0;s<this._cols.length;s++){if(s===t)continue;let i=n[s];if(!i||i.size===0)continue;let u=0;for(let g of r)i.has(g)&&u++;let d=Math.round(u/r.size*100);d>0&&a.push({otherColumn:this._cols[s].name,percentShared:d})}a.length>0&&(this._cols[t].sharedValueRatioWithOthers=a)}let N=50,l=[];for(let t=0;t<this._cols.length;t++){let r=n[t];r&&r.size>=2&&r.size<=N&&l.push(t)}if(l.length>=2){let t=new Map,r=(s,i)=>s+":"+i;for(let s=0;s<l.length;s++)for(let i=s+1;i<l.length;i++)t.set(r(l[s],l[i]),new Map);for(let s of this._rows)for(let i=0;i<l.length;i++){let u=this.STR(s[l[i]])+"";for(let d=i+1;d<l.length;d++){let g=""+this.STR(s[l[d]]),y=t.get(r(l[i],l[d])),w=u+""+g;y.set(w,(y.get(w)??0)+1)}}let a=new Map;for(let s=0;s<l.length;s++)for(let i=s+1;i<l.length;i++){let u=l[s],d=l[i],g=n[u].size,y=n[d].size,w=t.get(r(u,d)),R=w.size,A=0;for(let m of w.values())(A===0||m<A)&&(A=m);let v=Math.round(R/(g*y)*100),S=R===g,C=R===y;if((this._cols[u].categoricalPairStats||=[]).push({otherColumn:this._cols[d].name,fillPct:v,determinesOther:S,minCellCount:A}),(this._cols[d].categoricalPairStats||=[]).push({otherColumn:this._cols[u].name,fillPct:v,determinesOther:C,minCellCount:A}),S&&g>y){let m=a.get(u);(!m||y<m.card)&&a.set(u,{name:this._cols[d].name,card:y,ratio:y/g})}if(C&&y>g){let m=a.get(d);(!m||g<m.card)&&a.set(d,{name:this._cols[u].name,card:g,ratio:g/y})}}for(let[s,i]of a)this._cols[s].primaryParentColumn=i.name,this._cols[s].nestingRatio=Math.round(i.ratio*100)/100}return this._computedStatsForLevel=e,this._cols}updateColumnStats20(e,o,c,f,n,b,I,T,M,D,h,N){if(e>=20){let l=[...o].sort((a,s)=>a-s),t=Math.floor(l.length/2);if(c>0&&f!=null?n.avgValue=n.dataType=="Integer"?Math.round(f/c):f/c:n.avgValue=null,c>0&&(n.avgLength=b/c),this._rows.length>0&&(n.dataType!="DateTime"||I||!T?n.lowValue=T:n.lowValue=new Date(T).toISOString().slice(0,10)+"T00:00:00.000Z",n.dataType!="DateTime"||I||!M?n.highValue=M:n.highValue=new Date(M).toISOString().slice(0,10)+"T00:00:00.000Z",n.medianValue=l.length===0?null:l.length%2===0?n.dataType=="Integer"?Math.round((l[t-1]+l[t])/2):(l[t-1]+l[t])/2:l[t],n.numericPrecision=D),l.length>0){let a=n.avgValue??0,s=l.length,i=0,u=0;for(let g of l){let y=g-a;i+=y*y,u+=y*y*y}let d=i/s;n.skewness=d===0?0:u/s/Math.pow(Math.sqrt(d),3),n.stdDev=Math.sqrt(d)}f!==null&&(n.sum=f);let r=Array.from(h.entries()).sort((a,s)=>s[1]-a[1]);if(n.topCategories=r.slice(0,10).map(a=>a[1]),!n.isMeasure&&r.length>0){if(n.dataType==="String"){let s=r.map(u=>u[0]);n.formatSignature=H(s);let i=/^(UUID|EMAIL_LIKE|URL_LIKE|PHONE_LIKE|NUMERIC_ID|HEX_ID|OPAQUE_ID_ALPHANUMERIC|DATE_STR|BOOLEAN)$/;n.formatSignature&&!i.test(n.formatSignature)&&(n.avgLength===void 0||n.avgLength>=3)&&(n.isFreeText=!0)}else n.topCategoryValues=r.slice(0,5).map(s=>s[0]);if(e>=20){let s=r.map(u=>u[0]),i=k(s,N);if(i!==null)n.ordinalPattern=i.pattern,n.orderedDomain=i.orderedDomain;else if(n.dataType==="String"){let u=F(s);u!==null&&(n.safeDistinctValues=u)}}let a=U(r.map(s=>s[0]),n.name,N);a!==null&&(n.geoKind=a.geoKind,n.geoMatchPct=a.geoMatchPct,a.geoAmbiguous&&(n.geoAmbiguous=!0))}if(l.length>=8&&(n.dataType==="Integer"||n.dataType==="Decimal")){let a=s=>l[Math.floor((l.length-1)*s)];n.quantiles={p05:a(.05),p25:a(.25),p75:a(.75),p95:a(.95)}}}}updateColumnStats10(e,o,c,f,n,b,I){e>=10&&(o.blankCount=this._rows.length-c,o.distinctCount=f.size+(o.blankCount!=0?1:0),o.valueNature="Categorical",c>0&&(o.dataType==="Integer"||o.dataType==="Decimal"?o.valueNature=B({dataType:o.dataType,isMeasure:!!o.isMeasure,name:o.name,distinct:f.size,nonblank:c,prec:n,maxval:b,minval:I}):o.dataType==="DateTime"&&(o.valueNature=o.dateWithTime?"Continuous":"Ordinal")))}getRowCount(){return this._rows.length}getLeafCardinality(){if(this._leafCardinality!==null)return this._leafCardinality;let e=[];if(this._cols.forEach((c,f)=>{c.isMeasure||e.push(f)}),e.length===0)return this._leafCardinality=this._rows.length,this._leafCardinality;let o=new Set;for(let c of this._rows)o.add(JSON.stringify(e.map(f=>c[f])));return this._leafCardinality=o.size,this._leafCardinality}_leafCardinality=null;obfuscateString(e){let o="abcdefghijklmnopqrstuvwxyz",c=()=>o[Math.floor(x()*o.length)],f=()=>c().toUpperCase(),n=()=>Math.floor(x()*10).toString(),b=t=>/[A-Z]/.test(t)?f():/[a-z]/.test(t)?c():/[0-9]/.test(t)?n():t,I=e.includes("@")&&e.includes("."),T=I?e.split("@"):[e],M=I?T[1]:"",D=M.endsWith(".com"),h=D?".com":"",N=D?M.slice(0,-4):M,l=t=>{let r="";for(let a of t)r+=b(a),/[a-zA-Z0-9]/.test(a)&&x()<.2&&(r+=b(a));if(r.length>6){let a=Math.floor(x()*Math.min(3,r.length-6));for(let s=0;s<a;s++){let i=0;for(;i<5;){let u=Math.floor(x()*r.length);if(/[a-zA-Z0-9]/.test(r[u])){r=r.slice(0,u)+r.slice(u+1);break}i++}}}return r};if(I){let t=l(T[0]),r=l(N);return`${t}@${r}${h}`}else return l(e)}toObjectArray(){return this._rows.map(e=>{let o={};return this._cols.forEach((c,f)=>{o[c.name]=e[f]}),o})}async getCSVAsync(e){let o=[],c=e?25:1e7,f=e?4e3:1e7,n=new Map;for(let I of this._rows){let T={};for(let M=0;M<this._cols.length;M++){let D=this._cols[M],h=I[M];if(this.STR(h)!=""){if(e&&D.dataType=="String")if(n.has(h))h=n.get(h);else{let N=this.obfuscateString(h);n.set(h,N),h=N}else D.dataType=="DateTime"&&!D.dateWithTime&&(h=new Date(h).toISOString().slice(0,10)+"T00:00:00.000Z");f-=this.STR(h).length}else h=null;T[M]=h,--f}if(o.push(T),--c,c<=0||f<=0)break}return await X.unparse(o,{header:!1})}getCSVHeaderLine(){return X.unparse([this._cols.map(e=>this.STR(e.name))])}addRow(e,o){let c="";for(let n of e)c+=this.STR(n)+"~";let f=G(c);return this._rowHashes.has(f)?!1:(this._rows.push(e),this._origIndices.push(o??-1),this._rowHashes.add(f),!0)}getOriginalRowIndex(e){return e<0||e>=this._origIndices.length?-1:this._origIndices[e]}reset(){this._cols=[],this._rows=[],this._origIndices=[],this._rowHashes=new Set,this._leafCardinality=null}setColumns(e){this._cols=e}};export{at as a,ct as b,mt as c,ft as d,B as e,K as f};
|
|
2
|
+
import{a as Y,b as G,c as x}from"./chunk-NWDPA7DG.mjs";import{a as U}from"./chunk-GY54KQLP.mjs";import{a as z,b as F,c as k}from"./chunk-WBEROMBG.mjs";import{a as H}from"./chunk-QEH7WTLB.mjs";import{a as j,b as q}from"./chunk-2LR73DNS.mjs";import X from"papaparse";var Z=12,$=100,J=.3,Q=.9,tt=120,et=1900,nt=2100,it=0,st=20,rt=new Set(["id","ids","uuid","guid","key","zip","zipcode","postal","postalcode","ssn","code","codes","sku","isbn","msisdn","imei"]);function at(p){if(!p)return!1;let e=p.replace(/([a-z0-9])([A-Z])/g,"$1 $2").replace(/[_\-./]+/g," ").toLowerCase().split(/\s+/).filter(o=>o.length>0);return e.length===0?!1:rt.has(e[e.length-1])}var ot=/\b(year|yr|fy|fiscal\s*year|calendar\s*year)\b/i,ut=/^(?:(?:19|20)\d{2}(?:[-/ ]?(?:q[1-4]|w(?:0?[1-9]|[1-4]\d|5[0-3])|(?:0?[1-9]|1[0-2])))?|q[1-4][-/ ](?:19|20)\d{2}|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*[-/ ]?(?:19|20)?\d{2})$/i;function W(p){if(!Number.isInteger(p)||p<190001||p>210012)return!1;let e=p%100;return e>=1&&e<=12}function P(p){if(!Number.isInteger(p)||p<19000101||p>21001231)return!1;let e=Math.floor(p/100)%100,o=p%100;return e>=1&&e<=12&&o>=1&&o<=31}function lt(p,e){if(!e||!/\b(?:19|20)\d{2}\b/.test(p))return!1;let o=p.replace(/\b(?:19|20)\d{2}\b/g,"").replace(/[-/.,]/g," ").trim();if(!o)return!1;let c=j(o);return c.length>0&&e[c]!==void 0}function ct(p){let{dataType:e,name:o,isMeasure:c,distinctCount:f}=p;if(e==="DateTime")return!0;if(c)return!1;if(e==="Integer"||e==="Decimal"){let n=p.minNum,b=p.maxNum;if(!(n!=null&&b!=null&&Number.isFinite(n)&&Number.isFinite(b)&&Number.isInteger(n)&&Number.isInteger(b)&&b>=n))return!1;let T=n>=1900&&b<=2100;if(T&&f>=2&&ot.test(o)||T&&f>=3&&f===b-n+1)return!0;let M=W(n)&&W(b),D=P(n)&&P(b);if(M||D){if(!p.sampleValues)return!0;let h=0,N=0;for(let l of p.sampleValues){if(l==null||l==="")continue;h++;let t=Number(String(l).trim());if(Number.isFinite(t)&&(M?W(t):P(t))&&N++,h>=60)break}if(h>=2&&N===h)return!0}return!1}if(p.sampleValues){let n=p.locale?q(p.locale):null,b=0,I=0;for(let T of p.sampleValues){if(T==null||T==="")continue;b++;let M=String(T).trim();if((ut.test(M)||lt(M,n))&&I++,b>=60)break}if(b>=2&&I/b>=.8)return!0}return!1}function mt(p){let e=(p||"").trim().toLowerCase();return e.startsWith("sum ")||e.startsWith("total ")?"sum":e.startsWith("average ")||e.startsWith("avg. ")?"avg":e.startsWith("count ")||e.startsWith("distinct count ")||e.startsWith("count (distinct) ")?"count":e.startsWith("min ")||e.startsWith("minimum ")?"min":e.startsWith("max ")||e.startsWith("maximum ")?"max":null}function ft(p){let{rows:e,measureColIdx:o,measureName:c,candidateDims:f,minNum:n,maxNum:b,discourageAgg:I}=p,T=100,M=.01,D="";for(let N of f){if(N.distinct<2||N.distinct>T)continue;let l=f.filter(i=>i.idx!==N.idx).map(i=>i.idx),t=new Map;for(let i of e){let u=i[o];if(typeof u!="number"||!Number.isFinite(u))continue;let d=l.length?l.map(y=>String(i[y])).join(D):"",g=t.get(d);g||(g={sum:0,cset:new Set},t.set(d,g)),g.sum+=u,g.cset.add(String(i[N.idx]))}let r=0,a=0,s=0;for(let i of t.values())i.cset.size<2||(s++,Math.abs(i.sum-100)<=100*M?r++:Math.abs(i.sum-1)<=1*M&&a++);if(s>=2&&(r/s>=.9||a/s>=.9))return{additivity:"part_of_whole",partOfWholeDim:N.name}}if(I)return{additivity:"intensive_rate"};let h=mt(c);return h==="avg"||h==="min"||h==="max"?{additivity:"intensive_rate"}:h==="sum"||h==="count"?{additivity:"additive"}:n!=null&&b!=null&&Number.isFinite(n)&&Number.isFinite(b)&&n>=0&&b<=1.0001&&b>0?{additivity:"intensive_rate"}:{additivity:"unknown"}}function B(p){let{dataType:e,isMeasure:o,name:c,distinct:f,nonblank:n,prec:b,maxval:I,minval:T}=p,M=I-T+1,D=M>0?f/M:1,h=n>0?f/n:0,N=e==="Decimal"||b>0;return at(c)?"Categorical":D>=Q&&(z(c)&&f<=tt||T>=et&&I<=nt||T>=it&&I<=st)?"Ordinal":o||f>Z&&(N||f>=$||h>=J)?"Continuous":"Categorical"}var K=class{_computedStatsForLevel=null;_cols=[];_rows=[];_origIndices=[];_rowHashes=new Set;STR(e){return e==null?"":e.toString()}getPrecision(e){if(!isFinite(e))return 0;let c=e.toString().split(".")[1];return c?c.length:0}getColumns(){return this._cols}hasTimeComponent(e){return!(e instanceof Date)||isNaN(e.getTime())?!1:e.getHours()!==0||e.getMinutes()!==0||e.getSeconds()!==0||e.getMilliseconds()!==0}getColumnsWithStats(e,o){if(this._computedStatsForLevel==e)return this._cols;let c=0,f=parseInt(e),n=[];for(let t of this._cols){let r=new Map,a=[],s=0,i=0,u=0,d=null,g=null,y=null,w=!1,A=!0,R=!0,v=null,S=!1;for(let C of this._rows){let m=C[c],_=this.STR(m);if(_!=""){if(s+=_.length,r.set(_,(r.get(_)||0)+1),i++,t.dataType=="Integer"||t.dataType=="Decimal"||t.dataType=="DateTime")if((d===null||m<d)&&(d=m),(g===null||m>g)&&(g=m),t.dataType=="Integer"||t.dataType=="Decimal"){if(y===null?y=m:y+=m,t.dataType=="Decimal"){let L=this.getPrecision(m);L>u&&(u=L)}a.push(m)}else w=w||this.hasTimeComponent(m);S&&(m<v?A=!1:m>v&&(R=!1)),v=m,S=!0}}if(i>=3&&(A&&!R?t.sortedDirection="asc":R&&!A?t.sortedDirection="desc":t.sortedDirection="none"),i>0&&r.size>0&&!t.isMeasure){let C=0,m=Number.MAX_SAFE_INTEGER;for(let _ of r.values())_>C&&(C=_),_<m&&(m=_);if(t.modalShare=C/i,t.minGroupCount=m===Number.MAX_SAFE_INTEGER?null:m,r.size===2){let _=[...r.keys()].map(V=>Y(V).trim().toLowerCase()).sort(),E=[["false","true"],["no","yes"],["n","y"],["0","1"],["f","t"]].some(V=>_[0]===V[0]&&_[1]===V[1]),O=/^(is|has)[_a-z0-9]|(?:default|churn|fraud|approv|convert|active|cancel|delinquen|flag|paid|win|pass|fail)/i.test(t.name);(E||O)&&(t.isBinaryFlag=!0)}}t.dataType=="DateTime"&&(t.dateWithTime=w),t.isTemporal=ct({dataType:t.dataType,name:t.name,isMeasure:t.isMeasure,distinctCount:r.size,minNum:typeof d=="number"?d:null,maxNum:typeof g=="number"?g:null,sampleValues:r.keys(),locale:o}),this.updateColumnStats10(f,t,i,r,u,g,d),this.updateColumnStats20(f,a,i,y,t,s,w,d,g,u,r,o),!t.isMeasure&&(t.dataType==="Integer"||t.dataType==="Decimal")&&B({dataType:t.dataType,isMeasure:!1,name:t.name,distinct:r.size,nonblank:i,prec:u,maxval:typeof g=="number"?g:0,minval:typeof d=="number"?d:0})==="Continuous"&&(t.isMeasure=!0),n[c]=t.isMeasure||r.size>2e3?null:new Set(r.keys()),c++}let b=[];this._cols.forEach((t,r)=>{t.isMeasure||b.push({idx:r,distinct:t.distinctCount??0,name:t.name})}),this._cols.forEach((t,r)=>{if(!t.isMeasure)return;let a=typeof t.lowValue=="number"?t.lowValue:null,s=typeof t.highValue=="number"?t.highValue:null,i=ft({rows:this._rows,measureColIdx:r,measureName:t.name,candidateDims:b,minNum:a,maxNum:s,discourageAgg:!!t.discourageAggregationAcrossGroups});t.additivity=i.additivity,i.partOfWholeDim&&(t.partOfWholeDim=i.partOfWholeDim)});let I=50,T=b.filter(t=>t.distinct>=2&&t.distinct<=I);this._cols.forEach((t,r)=>{if(!t.isMeasure)return;let a=[];for(let v of this._rows){let S=v[r];if(S==null||S==="")continue;let C=typeof S=="number"?S:parseFloat(S);isNaN(C)||a.push(C)}if(a.length===0)return;let s=[...a].sort((v,S)=>v-S),i=v=>s[Math.min(s.length-1,Math.max(0,Math.floor(v*(s.length-1))))],u=i(.5),d=i(.1),g=i(.9),y=Math.abs(u)>1e-9?Math.abs(u):Math.abs(s[s.length-1])>1e-9?Math.abs(s[s.length-1]):1;t.relativeDispersion=Math.round((g-d)/y*1e3)/1e3;let w=0;for(let v of a)w+=v;w/=a.length;let A=0;for(let v of a)A+=(v-w)*(v-w);if(A<=1e-12)return;let R=[];for(let v of T){let S=new Map;for(let m of this._rows){let _=m[r];if(_==null||_==="")continue;let L=typeof _=="number"?_:parseFloat(_);if(isNaN(L))continue;let E=this.STR(m[v.idx])+"",O=S.get(E);O?(O.s+=L,O.n++):S.set(E,{s:L,n:1})}let C=0;for(let m of S.values()){let _=m.s/m.n;C+=m.n*(_-w)*(_-w)}R.push({otherColumn:v.name,eta2:Math.round(C/A*1e4)/1e4})}R.length>0&&(t.groupDiscrimination=R)});let M=.97,D=5,h=this._cols.map((t,r)=>({c:t,i:r})).filter(t=>t.c.isMeasure);for(let t=0;t<h.length;t++)for(let r=t+1;r<h.length;r++){let a=h[t].i,s=h[r].i,i=0,u=0,d=0,g=0,y=0,w=0;for(let S of this._rows){let C=S[a],m=S[s];if(C==null||C===""||m===null||m===void 0||m==="")continue;let _=typeof C=="number"?C:parseFloat(C),L=typeof m=="number"?m:parseFloat(m);isNaN(_)||isNaN(L)||(i++,u+=_,d+=L,g+=_*_,y+=L*L,w+=_*L)}if(i<D)continue;let A=i*g-u*u,R=i*y-d*d;if(A<=1e-12||R<=1e-12)continue;let v=(i*w-u*d)/Math.sqrt(A*R);if(Math.abs(v)>=M){let S=h[t].c,C=h[r].c;S.collinearWithMeasures||(S.collinearWithMeasures=[]),C.collinearWithMeasures||(C.collinearWithMeasures=[]),S.collinearWithMeasures.push(C.name),C.collinearWithMeasures.push(S.name)}}for(let t=0;t<this._cols.length;t++){let r=n[t];if(!r||r.size===0)continue;let a=[];for(let s=0;s<this._cols.length;s++){if(s===t)continue;let i=n[s];if(!i||i.size===0)continue;let u=0;for(let g of r)i.has(g)&&u++;let d=Math.round(u/r.size*100);d>0&&a.push({otherColumn:this._cols[s].name,percentShared:d})}a.length>0&&(this._cols[t].sharedValueRatioWithOthers=a)}let N=50,l=[];for(let t=0;t<this._cols.length;t++){let r=n[t];r&&r.size>=2&&r.size<=N&&l.push(t)}if(l.length>=2){let t=new Map,r=(s,i)=>s+":"+i;for(let s=0;s<l.length;s++)for(let i=s+1;i<l.length;i++)t.set(r(l[s],l[i]),new Map);for(let s of this._rows)for(let i=0;i<l.length;i++){let u=this.STR(s[l[i]])+"";for(let d=i+1;d<l.length;d++){let g=""+this.STR(s[l[d]]),y=t.get(r(l[i],l[d])),w=u+""+g;y.set(w,(y.get(w)??0)+1)}}let a=new Map;for(let s=0;s<l.length;s++)for(let i=s+1;i<l.length;i++){let u=l[s],d=l[i],g=n[u].size,y=n[d].size,w=t.get(r(u,d)),A=w.size,R=0;for(let m of w.values())(R===0||m<R)&&(R=m);let v=Math.round(A/(g*y)*100),S=A===g,C=A===y;if((this._cols[u].categoricalPairStats||=[]).push({otherColumn:this._cols[d].name,fillPct:v,determinesOther:S,minCellCount:R,distinctCombinations:A}),(this._cols[d].categoricalPairStats||=[]).push({otherColumn:this._cols[u].name,fillPct:v,determinesOther:C,minCellCount:R,distinctCombinations:A}),S&&g>y){let m=a.get(u);(!m||y<m.card)&&a.set(u,{name:this._cols[d].name,card:y,ratio:y/g})}if(C&&y>g){let m=a.get(d);(!m||g<m.card)&&a.set(d,{name:this._cols[u].name,card:g,ratio:g/y})}}for(let[s,i]of a)this._cols[s].primaryParentColumn=i.name,this._cols[s].nestingRatio=Math.round(i.ratio*100)/100}return this._computedStatsForLevel=e,this._cols}updateColumnStats20(e,o,c,f,n,b,I,T,M,D,h,N){if(e>=20){let l=[...o].sort((a,s)=>a-s),t=Math.floor(l.length/2);if(c>0&&f!=null?n.avgValue=n.dataType=="Integer"?Math.round(f/c):f/c:n.avgValue=null,c>0&&(n.avgLength=b/c),this._rows.length>0&&(n.dataType!="DateTime"||I||!T?n.lowValue=T:n.lowValue=new Date(T).toISOString().slice(0,10)+"T00:00:00.000Z",n.dataType!="DateTime"||I||!M?n.highValue=M:n.highValue=new Date(M).toISOString().slice(0,10)+"T00:00:00.000Z",n.medianValue=l.length===0?null:l.length%2===0?n.dataType=="Integer"?Math.round((l[t-1]+l[t])/2):(l[t-1]+l[t])/2:l[t],n.numericPrecision=D),l.length>0){let a=n.avgValue??0,s=l.length,i=0,u=0;for(let g of l){let y=g-a;i+=y*y,u+=y*y*y}let d=i/s;n.skewness=d===0?0:u/s/Math.pow(Math.sqrt(d),3),n.stdDev=Math.sqrt(d)}f!==null&&(n.sum=f);let r=Array.from(h.entries()).sort((a,s)=>s[1]-a[1]);if(n.topCategories=r.slice(0,10).map(a=>a[1]),!n.isMeasure&&r.length>0){if(n.dataType==="String"){let s=r.map(u=>u[0]);n.formatSignature=H(s);let i=/^(UUID|EMAIL_LIKE|URL_LIKE|PHONE_LIKE|NUMERIC_ID|HEX_ID|OPAQUE_ID_ALPHANUMERIC|DATE_STR|BOOLEAN)$/;n.formatSignature&&!i.test(n.formatSignature)&&(n.avgLength===void 0||n.avgLength>=3)&&(n.isFreeText=!0)}else n.topCategoryValues=r.slice(0,5).map(s=>s[0]);if(e>=20){let s=r.map(u=>u[0]),i=k(s,N);if(i!==null)n.ordinalPattern=i.pattern,n.orderedDomain=i.orderedDomain;else if(n.dataType==="String"){let u=F(s);u!==null&&(n.safeDistinctValues=u)}}let a=U(r.map(s=>s[0]),n.name,N);a!==null&&(n.geoKind=a.geoKind,n.geoMatchPct=a.geoMatchPct,a.geoAmbiguous&&(n.geoAmbiguous=!0))}if(l.length>=8&&(n.dataType==="Integer"||n.dataType==="Decimal")){let a=s=>l[Math.floor((l.length-1)*s)];n.quantiles={p05:a(.05),p25:a(.25),p75:a(.75),p95:a(.95)}}}}updateColumnStats10(e,o,c,f,n,b,I){e>=10&&(o.blankCount=this._rows.length-c,o.distinctCount=f.size+(o.blankCount!=0?1:0),o.valueNature="Categorical",c>0&&(o.dataType==="Integer"||o.dataType==="Decimal"?o.valueNature=B({dataType:o.dataType,isMeasure:!!o.isMeasure,name:o.name,distinct:f.size,nonblank:c,prec:n,maxval:b,minval:I}):o.dataType==="DateTime"&&(o.valueNature=o.dateWithTime?"Continuous":"Ordinal")))}getRowCount(){return this._rows.length}getLeafCardinality(){if(this._leafCardinality!==null)return this._leafCardinality;let e=[];if(this._cols.forEach((c,f)=>{c.isMeasure||e.push(f)}),e.length===0)return this._leafCardinality=this._rows.length,this._leafCardinality;let o=new Set;for(let c of this._rows)o.add(JSON.stringify(e.map(f=>c[f])));return this._leafCardinality=o.size,this._leafCardinality}_leafCardinality=null;obfuscateString(e){let o="abcdefghijklmnopqrstuvwxyz",c=()=>o[Math.floor(x()*o.length)],f=()=>c().toUpperCase(),n=()=>Math.floor(x()*10).toString(),b=t=>/[A-Z]/.test(t)?f():/[a-z]/.test(t)?c():/[0-9]/.test(t)?n():t,I=e.includes("@")&&e.includes("."),T=I?e.split("@"):[e],M=I?T[1]:"",D=M.endsWith(".com"),h=D?".com":"",N=D?M.slice(0,-4):M,l=t=>{let r="";for(let a of t)r+=b(a),/[a-zA-Z0-9]/.test(a)&&x()<.2&&(r+=b(a));if(r.length>6){let a=Math.floor(x()*Math.min(3,r.length-6));for(let s=0;s<a;s++){let i=0;for(;i<5;){let u=Math.floor(x()*r.length);if(/[a-zA-Z0-9]/.test(r[u])){r=r.slice(0,u)+r.slice(u+1);break}i++}}}return r};if(I){let t=l(T[0]),r=l(N);return`${t}@${r}${h}`}else return l(e)}toObjectArray(){return this._rows.map(e=>{let o={};return this._cols.forEach((c,f)=>{o[c.name]=e[f]}),o})}async getCSVAsync(e){let o=[],c=e?25:1e7,f=e?4e3:1e7,n=new Map;for(let I of this._rows){let T={};for(let M=0;M<this._cols.length;M++){let D=this._cols[M],h=I[M];if(this.STR(h)!=""){if(e&&D.dataType=="String")if(n.has(h))h=n.get(h);else{let N=this.obfuscateString(h);n.set(h,N),h=N}else D.dataType=="DateTime"&&!D.dateWithTime&&(h=new Date(h).toISOString().slice(0,10)+"T00:00:00.000Z");f-=this.STR(h).length}else h=null;T[M]=h,--f}if(o.push(T),--c,c<=0||f<=0)break}return await X.unparse(o,{header:!1})}getCSVHeaderLine(){return X.unparse([this._cols.map(e=>this.STR(e.name))])}addRow(e,o){let c="";for(let n of e)c+=this.STR(n)+"~";let f=G(c);return this._rowHashes.has(f)?!1:(this._rows.push(e),this._origIndices.push(o??-1),this._rowHashes.add(f),!0)}getOriginalRowIndex(e){return e<0||e>=this._origIndices.length?-1:this._origIndices[e]}reset(){this._cols=[],this._rows=[],this._origIndices=[],this._rowHashes=new Set,this._leafCardinality=null}setColumns(e){this._cols=e}};export{at as a,ct as b,mt as c,ft as d,B as e,K as f};
|
package/dist/index.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/*! @bicharts/shape-core — Apache-2.0. Bundled reference data: GeoNames (https://www.geonames.org/) CC BY 4.0; US Census/TIGER (public domain). Full text: NOTICE in this package. */
|
|
2
|
-
import{a as U,b as X,c as Y,d as tt,e as nt,f as et}from"./chunk-
|
|
2
|
+
import{a as U,b as X,c as Y,d as tt,e as nt,f as et}from"./chunk-FVSPBUW2.mjs";import{a as j,b as J,c as Q}from"./chunk-NWDPA7DG.mjs";import{a as K,b as V,c as E,d as F}from"./chunk-GY54KQLP.mjs";import{a as C,b as f,c as $,d as D,e as g,f as H,g as O,h as b,i as B}from"./chunk-3UOCZ2D6.mjs";import{a as G,b as N,c as _}from"./chunk-WBEROMBG.mjs";import{a as q}from"./chunk-QEH7WTLB.mjs";import{a as W,b as Z}from"./chunk-2LR73DNS.mjs";var R=85,S=2,ot=100,L=400,w=new Set(["us","usa","u s","u s a","united states","united states of america","america","ca","can","canada","mx","mex","mexico","estados unidos mexicanos"]);function m(i){let n=new Set;for(let e of i){if(e==null)continue;let o=C(String(e));if(o&&!n.has(o)&&(n.add(o),n.size>=L))break}return Array.from(n)}function P(i){let n=m(i);return n.length===0||!n.every(e=>w.has(e))?!1:n.some(e=>!f(e))}function M(i){return w.has(i)&&!!f(i)}function x(i){let n=m(i);if(n.length===0)return 0;let e=0;for(let o of n)f(o)&&e++;return Math.round(e/n.length*1e3)/10}function z(i){let n=new Set;for(let e of i){if(e==null)continue;let o=String(e).trim();if(o&&!n.has(o)&&(n.add(o),n.size>=L))break}return Array.from(n)}function I(i){let n=z(i);if(n.length===0)return 0;let e=0;for(let o of n)g(o).length>0&&e++;return Math.round(e/n.length*1e3)/10}function it(i){let n=m(i);if(n.length===0)return 0;let e=0;for(let o of n)b(o)&&e++;return Math.round(e/n.length*1e3)/10}function k(i,n){let e=0;for(let o of m(i))n(o)&&e++;return e}function st(i,n){let e=0;for(let o of z(i))n(o)&&e++;return e}function rt(i,n,e){let o=[],p=[],s={},A=new Set(i),d=t=>n.map(r=>r[t]);for(let t of["city","zip","lat","lon","country"]){let r=e?.[t];r&&A.has(r)&&(s[t]=r)}let u=e?.state;if(u&&A.has(u)){let t=d(u),r=m(t);r.length>0&&r.every(M)?p.push(`state=${u} (every value is "CA", which is both Canada and California)`):P(t)?(p.push(`state=${u} (every value is a country, not a state)`),s.country||(s.country=u)):x(t)<R?p.push(`state=${u} (only ${x(t)}% of values are states/provinces)`):s.state=u}let l=new Set(Object.values(s).filter(Boolean)),y=i.filter(t=>!l.has(t)&&!t.startsWith("__"));if(!s.state){let t=null;for(let r of y){let a=d(r);if(P(a))continue;let c=x(a);if(c<R)continue;let v=k(a,h=>!!f(h)),T=k(a,h=>!!f(h)&&!M(h));v<S&&T<1||(!t||c>t.pct)&&(t={name:r,pct:c})}t&&(s.state=t.name,l.add(t.name),o.push(`state=${t.name} (${t.pct}% states/provinces)`))}if(!s.zip)for(let t of y){if(l.has(t))continue;let r=d(t),a=I(r);if(!(a<ot)&&!(st(r,c=>g(c).length>0)<S)){s.zip=t,l.add(t),o.push(`zip=${t} (${a}% ZIP codes)`);break}}if(!s.country){for(let t of y)if(!l.has(t)&&P(d(t))){s.country=t,l.add(t),o.push(`country=${t}`);break}}if(!s.city){let t=null;for(let r of y){if(l.has(r))continue;let a=d(r),c=it(a);c<R||k(a,v=>b(v))<S||(!t||c>t.pct)&&(t={name:r,pct:c})}t&&(s.city=t.name,o.push(`city=${t.name} (${t.pct}% known cities)`))}return{bind:!!(s.city||s.state||s.zip||s.lat&&s.lon)?s:null,backfilled:o,refused:p}}export{Q as GET_RANDOM,et as IndexedText,J as SIMPLE_STRING_HASH,j as STR,x as admin1MatchPct,F as buildGeoIsoColumn,O as buildGeoPointColumns,B as cityMatchPct,tt as classifyAdditivity,nt as classifyNumericValueNature,X as classifyTemporal,q as detectFormatSignature,K as detectGeo,_ as detectOrdinalDomain,Y as hostAggHint,U as isIdentifierName,E as isJoinGeoKind,G as isOrdinalFriendlyName,P as looksLikeCountryColumn,Z as monthLookupFor,$ as normalizeCountry,W as normalizeMonthKey,C as normalizePlaceName,f as resolveAdmin1,H as resolveGeoPoint,rt as resolvePointRoles,N as safeDistinctValuesToShip,V as toGeoIso,I as zipMatchPct,g as zipPrefixCandidates,D as zipToPrefix3};
|
package/dist/indexedText.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/*! @bicharts/shape-core — Apache-2.0. Bundled reference data: GeoNames (https://www.geonames.org/) CC BY 4.0; US Census/TIGER (public domain). Full text: NOTICE in this package. */
|
|
2
|
-
import{a,b,c,d,e,f}from"./chunk-
|
|
2
|
+
import{a,b,c,d,e,f}from"./chunk-FVSPBUW2.mjs";import"./chunk-NWDPA7DG.mjs";import"./chunk-GY54KQLP.mjs";import"./chunk-3UOCZ2D6.mjs";import"./chunk-WBEROMBG.mjs";import"./chunk-QEH7WTLB.mjs";import"./chunk-2LR73DNS.mjs";export{f as IndexedText,d as classifyAdditivity,e as classifyNumericValueNature,b as classifyTemporal,c as hostAggHint,a as isIdentifierName};
|
package/dist/types/models.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bicharts/shape-core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.15",
|
|
4
4
|
"description": "Host-agnostic data-shape profiler: ingest a dataset's rows through IValueCollection.addRow and read back the measured shape as an LLMColumnWithValue[] payload — types, cardinality, temporal/ordinal detection, geographic region and point resolution, cross-column signals. MEASUREMENT only: the policy that decides what a measured shape means for chart selection lives server-side.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|