@sasjs/utils 2.49.0 → 2.49.1

package/macros/mp_makedata.sas

@@ -0,0 +1,107 @@
+/**
+  @file
+  @brief Create sample data based on the structure of an empty table
+  @details Many SAS projects involve sensitive datasets.  One way to _ensure_
+    the data is anonymised, is never to receive it in the first place!  Often
+    consultants are provided with empty tables, and expected to create complex
+    ETL flows.
+
+    This macro can help by taking an empty table, and populating it with data
+    according to the variable types and formats.
+
+    TODO:
+      @li Consider dates, datetimes, times, integers etc
+
+  Usage:
+
+      proc sql;
+      create table work.example(
+        TX_FROM float format=datetime19.,
+        DD_TYPE char(16),
+        DD_SOURCE char(2048),
+        DD_SHORTDESC char(256),
+        constraint pk primary key(tx_from, dd_type,dd_source),
+        constraint nnn not null(DD_SHORTDESC)
+      );
+      %mp_makedata(work.example)
+
+  @param [in] libds The empty table (libref.dataset) in which to create data
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_getvarlen.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_nobs.sas
+  @li mp_getcols.sas
+  @li mp_getpk.sas
+
+  <h4> Related Macros </h4>
+  @li mp_makedata.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_makedata(libds
+  ,obs=500
+  ,seed=1
+)/*/STORE SOURCE*/;
+
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
+
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
+  %put &sysmacroname: &libds has data, it will not be recreated;
+  %return;
+%end;
+
+/* set up temporary vars */
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select coalescec(pk_fields,'_all_') into: pk_fields from &ds1;
+
+data &ds2;
+  if 0 then set &libds;
+  do _n_=1 to &obs;
+    %let charvars=%mf_getvarlist(&libds,typefilter=C);
+    %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
+      %let col=%scan(&charvars,&i);
+      /* create random value based on observation number and colum length */
+      &col=repeat(put(md5(cats(_n_)),$hex32.),%mf_getvarlen(&libds,&col)/32);
+    %end;
+
+    %let numvars=%mf_getvarlist(&libds,typefilter=N);
+    %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
+      %let col=%scan(&numvars,&i);
+      &col=_n_;
+    %end;
+    output;
+  end;
+  stop;
+run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
+run;
+
+proc append base=&libds data=&ds2;
+run;
+
+proc sql;
+drop table &ds1, &ds2;
+
+%mend mp_makedata;

package/macros/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),$hex32.)
+%mend mp_md5;

package/macros/mp_perflog.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Logs a message in a dataset every time it is invoked
+  @details If the dataset does not exist, it is created.
+  Usage:
+
+      %mp_perflog(started)
+      %mp_perflog()
+      %mp_perflog(startanew,libds=work.newdataset)
+      %mp_perflog(finished,libds=work.newdataset)
+      %mp_perflog(finished)
+
+
+  @param label Provide label to go into the control dataset
+  @param libds= Provide a dataset in which to store performance stats.  Default
+              name is <code>work.mp_perflog</code>;
+
+  @version 9.2
+  @author Allan Bowe
+  @source https://github.com/sasjs/core
+
+**/
+
+%macro mp_perflog(label,libds=work.mp_perflog
+)/*/STORE SOURCE*/;
+
+  %if not (%mf_existds(&libds)) %then %do;
+    data &libds;
+      length sysjobid $10 label $256 dttm 8.;
+      format dttm datetime19.3;
+      call missing(of _all_);
+      stop;
+    run;
+  %end;
+
+  proc sql;
+    insert into &libds
+      set sysjobid="&sysjobid"
+        ,label=symget('label')
+        ,dttm=%sysfunc(datetime());
+  quit;
+
+%mend mp_perflog;

package/macros/mp_prevobs.sas

@@ -0,0 +1,88 @@
+/**
+  @file
+  @brief Enables previous observations to be re-instated
+  @details Remembers the last X observations by storing them in a hash table.
+  Is a convenience over the use of lag() or retain, when an entire observation
+  needs to be restored.
+
+  This macro will also restore automatic variables (such as _n_ and _error_).
+
+  Example Usage:
+
+      data example;
+        set sashelp.class;
+        calc_var=_n_*3;
+        %* initialise hash and save from PDV ;
+        %mp_prevobs(INIT,history=2)
+        if _n_ =10 then do;
+          %* fetch previous but 1 record;
+          %mp_prevobs(FETCH,-2)
+          put _n_= name= age= calc_var=;
+          %* fetch previous record;
+          %mp_prevobs(FETCH,-1)
+          put _n_= name= age= calc_var=;
+          %* reinstate current record ;
+          %mp_prevobs(FETCH,0)
+          put _n_= name= age= calc_var=;
+        end;
+      run;
+
+  Result:
+
+  <img src="https://imgur.com/PSjHoET.png" alt="mp_prevobs sas" width="400"/>
+
+  Credit is made to `data _null_` for authoring this very helpful paper:
+  https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
+
+  @param action Either FETCH a current or previous record, or INITialise.
+  @param record The relative (to current) position of the previous observation
+    to return.
+  @param history= The number of records to retain in the hash table. Default=5
+  @param prefix= the prefix to give to the variables used to store the hash name
+    and index. Default=mp_prevobs
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_prevobs(action,record,history=5,prefix=mp_prevobs
+)/*/STORE SOURCE*/;
+%let action=%upcase(&action);
+%let prefix=%upcase(&prefix);
+%let record=%eval((&record+0) * -1);
+
+%if &action=INIT %then %do;
+
+  if _n_ eq 1 then do;
+    attrib &prefix._VAR length=$64;
+    dcl hash &prefix._HASH(ordered:'Y');
+    &prefix._KEY=0;
+    &prefix._HASH.defineKey("&prefix._KEY");
+    do while(1);
+      call vnext(&prefix._VAR);
+      if &prefix._VAR='' then leave;
+      if &prefix._VAR eq "&prefix._VAR" then continue;
+      else if &prefix._VAR eq "&prefix._KEY" then continue;
+      &prefix._HASH.defineData(&prefix._VAR);
+    end;
+    &prefix._HASH.defineDone();
+  end;
+  /* this part has to happen before FETCHing */
+  &prefix._KEY+1;
+  &prefix._rc=&prefix._HASH.add();
+  if &prefix._rc then putlog 'adding' &prefix._rc=;
+  %if &history>0 %then %do;
+    if &prefix._key>&history+1 then
+      &prefix._HASH.remove(key: &prefix._KEY - &history - 1);
+    if &prefix._rc then putlog 'removing' &prefix._rc=;
+  %end;
+%end;
+%else %if &action=FETCH %then %do;
+  if &record>&prefix._key then putlog "Not enough records in &Prefix._hash yet";
+  else &prefix._rc=&prefix._HASH.find(key: &prefix._KEY - &record);
+  if &prefix._rc then putlog &prefix._rc= " when fetching " &prefix._KEY=
+    "with record &record and " _n_=;
+%end;
+
+%mend mp_prevobs;

package/macros/mp_recursivejoin.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Returns all children from a hierarchy table for a specified parent
+  @details Where data stores hierarchies in a simple parent / child mapping,
+    it is not always straightforward to extract all the children for a
+    particular parent.  This problem is known as a recursive self join.  This
+    macro will extract all the descendents for a parent.
+  Usage:
+
+      data have;
+        p=1;c=2;output;
+        p=2;c=3;output;
+        p=2;c=4;output;
+        p=3;c=5;output;
+        p=6;c=7;output;
+        p=8;c=9;output;
+      run;
+
+      %mp_recursivejoin(base_ds=have
+        ,outds=want
+        ,matchval=1
+        ,parentvar=p
+        ,childvar=c
+        )
+
+  @param base_ds= base table containing hierarchy (not modified)
+  @param outds= the output dataset to create with the generated hierarchy
+  @param matchval= the ultimate parent from which to filter
+  @param parentvar= name of the parent variable
+  @param childvar= name of the child variable (should be same type as parent)
+  @param mdebug= set to 1 to prevent temp tables being dropped
+
+
+  @returns outds contains the following variables:
+    - level (0 = top level)
+    - &parentvar
+    - &childvar (null if none found)
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_recursivejoin(base_ds=
+    ,outds=
+    ,matchval=
+    ,parentvar=
+    ,childvar=
+    ,iter= /* reserved for internal / recursive use by the macro itself */
+    ,maxiter=500 /* avoid infinite loop */
+    ,mDebug=0);
+
+%if &iter= %then %do;
+  proc sql;
+  create table &outds as
+    select 0 as level,&parentvar, &childvar
+    from &base_ds
+    where &parentvar=&matchval;
+  %if &sqlobs.=0 %then %do;
+    %put NOTE: &sysmacroname: No match for &parentvar=&matchval;
+    %return;
+  %end;
+  %let iter=1;
+%end;
+%else %if &iter>&maxiter %then %return;
+
+proc sql;
+create table _data_ as
+  select &iter as level
+    ,curr.&childvar as &parentvar
+    ,base_ds.&childvar as &childvar
+  from &outds curr
+  left join &base_ds base_ds
+  on  curr.&childvar=base_ds.&parentvar
+  where curr.level=%eval(&iter.-1)
+    & curr.&childvar is not null;
+%local append_ds; %let append_ds=&syslast;
+%local obs; %let obs=&sqlobs;
+insert into &outds select distinct * from &append_ds;
+%if &mdebug=0 %then drop table &append_ds;;
+
+%if &obs %then %do;
+  %mp_recursivejoin(iter=%eval(&iter.+1)
+    ,outds=&outds,parentvar=&parentvar
+    ,childvar=&childvar
+    ,base_ds=&base_ds
+    )
+%end;
+
+%mend mp_recursivejoin;

package/macros/mp_replace.sas

@@ -0,0 +1,151 @@
+/**
+  @file
+  @brief Performs a text substitution on a file
+  @details Performs a find and replace on a file, either in place or to a new
+  file. Can be used on files where lines are longer than 32767.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the replace.
+
+  Full credit for this highly efficient and syntactically satisfying SAS logic
+  goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of
+  the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework.
+
+  Usage:
+
+      %let file="%sysfunc(pathname(work))/file.txt";
+      %let str=replace/me;
+      %let rep=with/this;
+      data _null_;
+        file &file;
+        put 'blahblah';
+        put "blahblah&str.blah";
+        put 'blahblahblah';
+      run;
+      %mp_replace(&file, findvar=str, replacevar=rep)
+      data _null_;
+        infile &file;
+        input;
+        list;
+      run;
+
+  Note - if you are running a version of SAS that will allow the io package in
+  LUA, you can also use this macro: mp_gsubfile.sas
+
+  @param infile The QUOTED path to the file on which to perform the substitution
+  @param findvar= Macro variable NAME containing the string to search for
+  @param replacevar= Macro variable NAME containing the replacement string
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (to
+    avoid overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_chop.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.test.sas
+
+  @version 9.4
+  @author Bartosz Jabłoński
+  @author Allan Bowe
+**/
+
+%macro mp_replace(infile,
+  findvar=,
+  replacevar=,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local inref dttm ds1;
+%let inref=%mf_getuniquefileref();
+%let outref=%mf_getuniquefileref();
+%if &outfile=0 %then %let outfile=&infile;
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &inref &infile lrecl=1 recfm=n;
+
+data &ds1;
+  infile &inref;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&findvar));
+  string =symget("&findvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+data &ds1;
+  declare hash HS(dataset:"&ds2(keep=start)");
+  HS.defineKey("start");
+  HS.defineDone();
+  declare hash HE(dataset:"&ds2(keep=stop)");
+  HE.defineKey("stop");
+  HE.defineDone();
+  do until(eof);
+    set &ds1 end=eof curobs =n;
+    start = ^HS.check(key:n);
+    stop  = ^HE.check(key:n);
+    length strt $ 1;
+    strt =put(start,best. -L);
+    retain out 1;
+    if out   then output;
+    if start then out=0;
+    if stop  then out=1;
+  end;
+  stop;
+  keep sourcechar strt;
+run;
+
+filename &outref &outfile recfm=n;
+
+data _null_;
+  length replace $ %length(%superq(&replacevar));
+  replace=symget("&replacevar");
+  file &outref;
+  do until(eof);
+    set &ds1 end=eof;
+    if strt ="1" then put replace char.;
+    else put sourcechar char1.;
+  end;
+  stop;
+run;
+
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_replace;

package/macros/mp_reseterror.sas

@@ -0,0 +1,27 @@
+/**
+  @file
+  @brief Reset when an err condition occurs
+  @details When building apps, sometimes an operation must be attempted that
+  can cause an err condition.  There is no try catch in SAS! So the err state
+  must be caught and reset.
+
+  This macro attempts to do that reset.
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_reseterror(
+)/*/STORE SOURCE*/;
+
+options obs=max replace nosyntaxcheck;
+%let syscc=0;
+
+%if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
+  data _null_;
+    rc=stpsrvset('program error', 0);
+  run;
+%end;
+
+%mend mp_reseterror;

package/macros/mp_resetoption.sas

@@ -0,0 +1,39 @@
+/**
+  @file
+  @brief Reset an option to original value
+  @details Inspired by the SAS Jedi -
+https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
+
+  Called as follows:
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
+
+
+  @param [in] option the option to reset
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_resetoption(option /* the option to reset */
+)/*/STORE SOURCE*/;
+
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  data _null_;
+    length code  $1500;
+    startup=getoption("&option",'startupvalue');
+    current=getoption("&option");
+    if startup ne current then do;
+      code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';');
+      putlog "NOTE: Resetting system option: " code ;
+      call execute(code );
+    end;
+  run;
+%end;
+%else %do;
+  %put &sysmacroname: reset option feature unavailable on &sysvlong;
+%end;
+%mend mp_resetoption;