@sasjs/utils 3.5.1 → 3.5.2

package/macros/mp_loadformat.sas

@@ -9,6 +9,9 @@
   format, to prevent loss of data - UNLESS the input dataset contains a marker
   column, specifying that a particular row needs to be deleted (`delete_col=`).
 
+  Positions of formats are made using the FMTROW variable - this must be present
+  and unique (on TYPE / FMTNAME / FMTROW).
+
   This macro can also be used to identify which records would be (or were)
   considered new, modified or deleted (`loadtarget=`) by creating the following
   tables:
@@ -17,7 +20,7 @@
   @li work.outds_del
   @li work.outds_mod
 
-  For example usage, see mp_loadformat.test.sas
+  For example usage, see test (under Related Macros)
 
   @param [in] libcat The format catalog to be loaded
   @param [in] libds The staging table to load
@@ -34,18 +37,22 @@
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
-  @li mddl_sas_cntlout.sas
+  @li mf_existds.sas
+  @li mf_existvar.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
+  @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_md5.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
-  @li mp_loadformat.test.sas
+  @li mp_loadformat.test.1.sas
+  @li mp_loadformat.test.2.sas
   @li mp_lockanytable.sas
   @li mp_stackdiffs.sas
 
@@ -67,7 +74,7 @@
 );
 /* set up local macro variables and temporary tables (with a prefix) */
 %local err msg prefix dslist i var fmtlist ibufsize;
-%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2;
 %if &outds_add=0 %then %let dslist=&dslist outds_add;
 %if &outds_del=0 %then %let dslist=&dslist outds_del;
 %if &outds_mod=0 %then %let dslist=&dslist outds_mod;
@@ -78,17 +85,20 @@
   %let &var=%upcase(&prefix._&var);
 %end;
 
-/*
-format values can be up to 32767 wide.  SQL joins on such a wide column can
-cause buffer issues.  Update ibufsize and reset at the end.
-*/
-%let ibufsize=%sysfunc(getoption(ibufsize));
-options ibufsize=32767 ;
-
 /* in DC, format catalogs maybe specified in the libds with a -FC extension */
 %let libcat=%scan(&libcat,1,-);
 
 /* perform input validations */
+%mp_abort(
+  iftrue=(%mf_existds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(&libds could not be found)
+)
+%mp_abort(
+  iftrue=(%mf_existvar(&libds,FMTROW)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(FMTROW not found in &libds)
+)
 %let err=0;
 %let msg=0;
 data _null_;
@@ -109,13 +119,6 @@ data _null_;
       stop;
     end;
   end;
-  else if name='LIBDS' then do;
-    if exist(value) le 0 then do;
-      call symputx('msg',"Unable to open staging table: "!!value);
-      call symputx('err',1);
-      stop;
-    end;
-  end;
   else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
   and missing(value) then do;
     call symputx('msg',"missing value in var: "!!name);
@@ -123,6 +126,14 @@ data _null_;
     stop;
   end;
 run;
+data _null_;
+  set &libds;
+  if missing(fmtrow) then do;
+    call symputx('msg',"missing fmtrow in format: "!!FMTNAME);
+    call symputx('err',1);
+    stop;
+  end;
+run;
 
 %mp_abort(
   iftrue=(&err ne 0)
@@ -130,33 +141,69 @@ run;
   ,msg=%str(&msg)
 )
 
+%local cnt;
+proc sql noprint;
+select count(distinct catx('|',type,fmtname,fmtrow)) into: cnt from &libds;
+%mp_abort(
+  iftrue=(&cnt ne %mf_nobs(&libds))
+  ,mac=&sysmacroname
+  ,msg=%str(Non-unique primary key on &libds)
+)
+
 /**
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
+select distinct
+  case
+    when type='N' then upcase(fmtname)
+    when type='C' then cats('$',upcase(fmtname))
+    when type='I' then cats('@',upcase(fmtname))
+    when type='J' then cats('@$',upcase(fmtname))
+    else "&sysmacroname:UNHANDLED"
+  end
+  into: fmtlist separated by ' '
+  from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
+/* get a hash of the row */
+%local cvars nvars;
+%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP
+  DIG3SEP DATATYPE LANGUAGE;
+%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
+data &base_fmts/note2err;
+  set &base_fmts;
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
-%mddl_sas_cntlout(libds=&template)
-data &inlibds;
-  length &delete_col $3;
-  if 0 then set &template;
+data &inlibds/nonote2err;
+  length &delete_col $3 FMTROW 8 start end label $32767;
+  if 0 then set &base_fmts;
   set &libds;
+  by type fmtname notsorted;
   if &delete_col='' then &delete_col='No';
   fmtname=upcase(fmtname);
+  type=upcase(type);
   if missing(type) then do;
-    if substr(fmtname,1,1)='$' then type='C';
-    else type='N';
+    if substr(fmtname,1,1)='@' then do;
+      if substr(fmtname,2,1)='$' then type='J';
+      else type='I';
+    end;
+    else do;
+      if substr(fmtname,1,1)='$' then type='C';
+      else type='N';
+    end;
   end;
-  if type='N' then do;
-    start=cats(start);
-    end=cats(end);
+  if type in ('N','I') then do;
+    %mp_aligndecimal(start,width=16)
+    %mp_aligndecimal(end,width=16)
   end;
+
+  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
 run;
 
 /**
@@ -167,11 +214,22 @@ create table &outds_add(drop=&delete_col) as
   select a.*
   from &inlibds a
   left join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where b.fmtname is null
     and upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;;
+  order by type, fmtname, fmtrow;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where upcase(a.&delete_col) ne "YES"
+    and a.fmthash ne b.fmthash
+  order by type, fmtname, fmtrow;
 
 /**
   * Identify deleted records
@@ -180,24 +238,28 @@ create table &outds_del(drop=&delete_col) as
   select a.*
   from &inlibds a
   inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
   where upcase(a.&delete_col)="YES"
-  order by fmtname, start;
+  order by type, fmtname, fmtrow;
 
 /**
-  * Identify modified records
+  * Identify fully deleted formats (where every record is removed)
+  * These require to be explicitly deleted in proc format
+  * del1 - identify _partial_ deletes
+  * del2 - exclude these, and also formats that come with _additions_
   */
-create table &outds_mod (drop=&delete_col) as
+create table &del1 as
   select a.*
-  from &inlibds a
-  inner join &base_fmts b
-  on a.fmtname=b.fmtname
-    and a.start=b.start
-  where upcase(a.&delete_col) ne "YES"
-  order by fmtname, start;
+  from &base_fmts a
+  left join &outds_del b
+  on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow
+  where b.fmtrow is null;
+
+create table &del2 as
+  select * from &outds_del
+  where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add)
+    and cats(type,fmtname) not in (select cats(type,fmtname) from &del1);
 
-options ibufsize=&ibufsize;
 
 %mp_abort(
   iftrue=(&syscc ne 0)
@@ -206,19 +268,21 @@ options ibufsize=&ibufsize;
 )
 
 %if &loadtarget=YES %then %do;
+  /* new records plus base records that are not deleted or modified */
   data &ds1;
     merge &base_fmts(in=base)
       &outds_mod(in=mod)
       &outds_add(in=add)
       &outds_del(in=del);
     if not del and not mod;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
+  /* add back the modified records */
   data &stagedata;
     set &ds1 &outds_mod;
   run;
   proc sort;
-    by fmtname start;
+    by type fmtname fmtrow;
   run;
 %end;
 /* mp abort needs to run outside of conditional blocks */
@@ -228,7 +292,7 @@ options ibufsize=&ibufsize;
   ,msg=%str(SYSCC=&syscc prior to actual load)
 )
 %if &loadtarget=YES %then %do;
-  %if %mf_nobs(&stagedata)=0 %then %do;
+  %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do;
     %put There are no changes to load in &libcat!;
     %return;
   %end;
@@ -244,6 +308,22 @@ options ibufsize=&ibufsize;
   /* do the actual load */
   proc format lib=&libcat cntlin=&stagedata;
   run;
+  /* apply any full deletes */
+  %if %mf_nobs(&del2)>0 %then %do;
+    %local delfmtlist;
+    proc sql noprint;
+    select distinct case when type='N' then cats(fmtname,'.FORMAT')
+        when type='C' then cats(fmtname,'.FORMATC')
+        when type='J' then cats(fmtname,'.INFMTC')
+        when type='I' then cats(fmtname,'.INFMT')
+        else cats(fmtname,'.BADENTRY!!!') end
+      into: delfmtlist
+      separated by ' '
+      from &del2;
+    proc catalog catalog=&libcat;
+      delete &delfmtlist;
+    quit;
+  %end;
   %if &locklibds ne 0 %then %do;
     /* unlock the table */
     %mp_lockanytable(UNLOCK
@@ -266,7 +346,7 @@ options ibufsize=&ibufsize;
 
     %mp_storediffs(&libcat-FC
       ,&base_fmts
-      ,FMTNAME START
+      ,TYPE FMTNAME FMTROW
       ,delds=&outds_del
       ,modds=&outds_mod
       ,appds=&outds_add

package/macros/mp_lockanytable.sas

@@ -167,7 +167,7 @@ run;
         data _null_;
           putlog 'NOTE-' / 'NOTE-';
           putlog "NOTE- &sysmacroname: table locked, waiting "@;
-          putlog "%sysfunc(sleep(&loop_inc)) seconds.. ";
+          putlog "%sysfunc(sleep(&loop_secs)) seconds.. ";
           putlog "NOTE- (iteration &x of &loops)";
           putlog 'NOTE-' / 'NOTE-';
         run;
@@ -194,32 +194,39 @@ run;
   %end;
 %end;
 %else %if &ACTION=UNLOCK %then %do;
-  %local status;
+  %local status cnt;
+  %let cnt=0;
   proc sql noprint;
-  select LOCK_STATUS_CD into: status from &ctl_ds
-    where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-  quit;
-  %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
-  %if &status=LOCKED %then %do;
-    data _null_;
-      putlog "&sysmacroname: unlocking &lib..&ds:";
-    run;
-    proc sql;
-    update &ctl_ds
-      set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
-        , LOCK_USER_NM="&user"
-        , LOCK_PID="&sysjobid"
-        , LOCK_REF="&ref"
-      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
-    quit;
-  %end;
-  %else %if &status=UNLOCKED %then %do;
-    %put %str(WAR)NING: &lib..&ds is already unlocked!;
+  select count(*) into: cnt from &ctl_ds where LOCK_LIB ="&lib" & LOCK_DS="&ds";
+  %if &cnt=0 %then %do;
+    %put %str(WAR)NING: &lib..&ds was not previously locked in &ctl_ds!;
   %end;
   %else %do;
-    %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
-    %let abortme=1;
+    select LOCK_STATUS_CD into: status from &ctl_ds
+      where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+    quit;
+    %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc;
+    %if &status=LOCKED %then %do;
+      data _null_;
+        putlog "&sysmacroname: unlocking &lib..&ds:";
+      run;
+      proc sql;
+      update &ctl_ds
+        set LOCK_STATUS_CD='UNLOCKED'
+          , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
+          , LOCK_USER_NM="&user"
+          , LOCK_PID="&sysjobid"
+          , LOCK_REF="&ref"
+        where LOCK_LIB ="&lib" and LOCK_DS="&ds";
+      quit;
+    %end;
+    %else %if &status=UNLOCKED %then %do;
+      %put %str(WAR)NING: &lib..&ds is already unlocked!;
+    %end;
+    %else %do;
+      %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds;
+      %let abortme=1;
+    %end;
   %end;
 %end;
 %else %do;

package/macros/mp_md5.sas

@@ -2,8 +2,7 @@
   @file
   @brief Generates an md5 expression for hashing a set of variables
   @details This is the same algorithm used to hash records in
-  [Data Controller for SAS](https://datacontroller.io) (free for up
-  to 5 users).
+  [Data Controller for SAS](https://datacontroller.io).
 
   It is not designed to be efficient - it is designed to be effective,
   given the range of edge cases (large floating points, special missing
@@ -29,8 +28,8 @@
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
-  @param cvars= Space seperated list of character variables
-  @param nvars= Space seperated list of numeric variables
+  @param [in] cvars= () Space seperated list of character variables
+  @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas

package/macros/mp_perflog.sas

@@ -11,9 +11,9 @@
       %mp_perflog(finished)
 
 
-  @param label Provide label to go into the control dataset
-  @param libds= Provide a dataset in which to store performance stats.  Default
-              name is <code>work.mp_perflog</code>;
+  @param [in] label Provide label to go into the control dataset
+  @param [in] libds= (work.mp_perflog) Provide a dataset in which to store
+    performance stats.  Default name is <code>work.mp_perflog</code>;
 
   @version 9.2
   @author Allan Bowe

package/macros/mp_prevobs.sas

@@ -34,12 +34,12 @@
   Credit is made to `data _null_` for authoring this very helpful paper:
   https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
 
-  @param action Either FETCH a current or previous record, or INITialise.
-  @param record The relative (to current) position of the previous observation
+  @param [in] action Either FETCH a current or previous record, or INITialise.
+  @param [in] record The relative (to current) position of the previous row
     to return.
-  @param history= The number of records to retain in the hash table. Default=5
-  @param prefix= the prefix to give to the variables used to store the hash name
-    and index. Default=mp_prevobs
+  @param [in] history= (5) The number of records to retain in the hash table.
+  @param [in] prefix= (mp_prevobs) The prefix to give to the variables used to
+    store the hash name and index.
 
   @version 9.2
   @author Allan Bowe

package/macros/mp_recursivejoin.sas

@@ -23,12 +23,13 @@
         ,childvar=c
         )
 
-  @param base_ds= base table containing hierarchy (not modified)
-  @param outds= the output dataset to create with the generated hierarchy
-  @param matchval= the ultimate parent from which to filter
-  @param parentvar= name of the parent variable
-  @param childvar= name of the child variable (should be same type as parent)
-  @param mdebug= set to 1 to prevent temp tables being dropped
+  @param [in] base_ds= base table containing hierarchy (not modified)
+  @param [out] outds= the output dataset to create with the generated hierarchy
+  @param [in] matchval= the ultimate parent from which to filter
+  @param [in] parentvar= name of the parent variable
+  @param [in] childvar= () name of the child variable (should be same type as
+    parent)
+  @param [in] mdebug= set to 1 to prevent temp tables being dropped
 
 
   @returns outds contains the following variables:

package/macros/mp_replace.sas

@@ -32,10 +32,11 @@
   Note - if you are running a version of SAS that will allow the io package in
   LUA, you can also use this macro: mp_gsubfile.sas
 
-  @param infile The QUOTED path to the file on which to perform the substitution
-  @param findvar= Macro variable NAME containing the string to search for
-  @param replacevar= Macro variable NAME containing the replacement string
-  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
+  @param [in] infile The QUOTED path to the file on which to perform the
+    substitution
+  @param [in] findvar= Macro variable NAME containing the string to search for
+  @param [in] replacevar= Macro variable NAME containing the replacement string
+  @param [out] outfile= (0) Optional QUOTED path to the adjusted output file (to
     avoid overwriting the first file).
 
   <h4> SAS Macros </h4>

package/macros/mp_runddl.sas

@@ -31,9 +31,9 @@
     %mp_runddl(/some/rootlib, exc=LIBREF3) * same as above ;
 
 
-  @param path location of the DDL folder structure
-  @param inc= list of librefs to include
-  @param exc= list of librefs to exclude (takes precedence over inc=)
+  @param [in] path location of the DDL folder structure
+  @param [in] inc= list of librefs to include
+  @param [in] exc= list of librefs to exclude (takes precedence over inc=)
 
   @version 9.3
   @author Allan Bowe

package/macros/mp_searchcols.sas

@@ -9,12 +9,13 @@
 
       %mp_searchcols(libs=sashelp work, cols=name sex age)
 
-  @param libs=(SASHELP) Space separated list of libraries to search for columns
-  @param cols= Space separated list of column names to search for (not case
-    sensitive)
-  @param outds=(mp_searchcols) the table to create with the results.  Will have
-    one line per table match.
-  @param match=(ANY) The match type. Valid values:
+  @param [in] libs= (SASHELP)
+    Space separated list of libraries to search for columns
+  @param [in] cols=
+    Space separated list of column names to search for (not case sensitive)
+  @param [out] outds= (mp_searchcols)
+    The table to create with the results.  Will have one line per table match.
+  @param [in] match= (ANY) The match type. Valid values:
     @li ANY - The table contains at least one of the columns
     @li WILD - The table contains a column with a name that partially matches
 

package/macros/mp_stackdiffs.sas

@@ -197,6 +197,7 @@
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
+  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using

package/macros/mp_storediffs.sas

@@ -6,8 +6,7 @@
     data recovery, and change re-application.  This macro is one of many
     data management utilities used in [Data Controller for SAS](
     https:datacontroller.io) - a comprehensive data ingestion solution, which
-    works on any SAS platform (Viya, SAS 9, Foundation) and is free for up to 5
-    users.
+    works on any SAS platform (Viya, SAS 9, Foundation).
 
     NOTE - this macro does not validate the inputs. It is assumed that the
     datasets containing the new / changed / deleted rows are CORRECT, contain
@@ -65,6 +64,7 @@
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
+  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -147,9 +147,9 @@ run;
 
 %if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
   /* this is a format catalog - cannot query cols directly */
-  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
-    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
-    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+  %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX"
+    ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL"
+    ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
 %end;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
@@ -165,7 +165,7 @@ data &ds4;
   if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
   else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
-    putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
+    putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);
   end;
 
@@ -174,7 +174,7 @@ data &ds4;
   else if &inds_keep="&modds" then move_type='M';
   else if &inds_keep="&origds" then move_type='O';
   else do;
-    putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
+    putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep;
     call symputx('syscc',99);
   end;
   tgtvar_nm=upcase(tgtvar_nm);
@@ -184,7 +184,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */

package/macros/mp_stprequests.sas

@@ -21,9 +21,10 @@
     %mp_stprequests(status_cd=INIT, libds=YOURLIB.DATASET )
 
 
-  @param status_cd= Use INIT for INIT and TERM for TERM events
-  @param libds= Location of base table (library.dataset).  To minimise risk
-    of table locks, we HIGHLY recommend using a database (NOT a SAS dataset).
+  @param [in] status_cd= Use INIT for INIT and TERM for TERM events
+  @param [in] libds= (somelib.stp_requests) Location of base table
+    (library.dataset).  To minimise risk of table locks, we HIGHLY recommend
+    using a database (NOT a SAS dataset).
     THE LIBRARY SHOULD BE ASSIGNED ALREADY - eg in autoexec or earlier in the
     init program proper.